2024-11-25 03:36:41,061 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-25 03:36:41,077 main DEBUG Took 0.013828 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-25 03:36:41,078 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-25 03:36:41,078 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-25 03:36:41,080 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-25 03:36:41,081 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:36:41,091 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-25 03:36:41,105 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:36:41,107 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:36:41,107 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:36:41,108 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:36:41,108 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:36:41,109 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:36:41,109 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:36:41,110 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:36:41,110 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:36:41,110 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:36:41,111 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:36:41,111 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:36:41,112 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:36:41,112 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:36:41,113 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:36:41,113 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:36:41,114 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:36:41,114 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:36:41,115 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:36:41,115 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:36:41,116 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:36:41,116 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:36:41,116 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:36:41,117 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:36:41,117 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:36:41,117 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-25 03:36:41,119 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:36:41,121 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-25 03:36:41,123 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-25 03:36:41,123 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-25 03:36:41,124 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-25 03:36:41,125 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-25 03:36:41,134 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-25 03:36:41,138 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-25 03:36:41,140 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-25 03:36:41,141 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-25 03:36:41,142 main DEBUG createAppenders(={Console}) 2024-11-25 03:36:41,143 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-25 03:36:41,143 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-25 03:36:41,144 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-25 03:36:41,144 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-25 03:36:41,145 main DEBUG OutputStream closed 2024-11-25 03:36:41,145 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-25 03:36:41,146 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-25 03:36:41,146 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-25 03:36:41,239 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-25 03:36:41,242 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-25 03:36:41,244 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-25 03:36:41,245 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-25 03:36:41,246 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-25 03:36:41,246 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-25 03:36:41,247 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-25 03:36:41,247 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-25 03:36:41,247 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-25 03:36:41,247 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-25 03:36:41,248 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-25 03:36:41,248 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-25 03:36:41,248 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-25 03:36:41,249 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-25 03:36:41,249 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-25 03:36:41,250 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-25 03:36:41,250 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-25 03:36:41,251 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-25 03:36:41,257 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-25 03:36:41,257 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-25 03:36:41,258 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-25 03:36:41,259 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-25T03:36:41,552 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efa42ab5-ac11-230a-67c9-32b4726a6320 2024-11-25 03:36:41,555 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-25 03:36:41,556 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-25T03:36:41,567 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-25T03:36:41,590 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-25T03:36:41,610 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efa42ab5-ac11-230a-67c9-32b4726a6320/cluster_2bccc8e6-c56b-5b8b-fc38-b8aab8adb7e9, deleteOnExit=true 2024-11-25T03:36:41,610 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-25T03:36:41,612 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efa42ab5-ac11-230a-67c9-32b4726a6320/test.cache.data in system properties and HBase conf 2024-11-25T03:36:41,612 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efa42ab5-ac11-230a-67c9-32b4726a6320/hadoop.tmp.dir in system properties and HBase conf 2024-11-25T03:36:41,613 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efa42ab5-ac11-230a-67c9-32b4726a6320/hadoop.log.dir in system properties and HBase conf 2024-11-25T03:36:41,614 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efa42ab5-ac11-230a-67c9-32b4726a6320/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-25T03:36:41,614 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efa42ab5-ac11-230a-67c9-32b4726a6320/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-25T03:36:41,615 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-25T03:36:41,724 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-25T03:36:41,836 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T03:36:41,842 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efa42ab5-ac11-230a-67c9-32b4726a6320/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-25T03:36:41,843 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efa42ab5-ac11-230a-67c9-32b4726a6320/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-25T03:36:41,843 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efa42ab5-ac11-230a-67c9-32b4726a6320/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-25T03:36:41,844 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efa42ab5-ac11-230a-67c9-32b4726a6320/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-25T03:36:41,845 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efa42ab5-ac11-230a-67c9-32b4726a6320/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-25T03:36:41,845 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efa42ab5-ac11-230a-67c9-32b4726a6320/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-25T03:36:41,846 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efa42ab5-ac11-230a-67c9-32b4726a6320/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-25T03:36:41,846 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efa42ab5-ac11-230a-67c9-32b4726a6320/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-25T03:36:41,847 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efa42ab5-ac11-230a-67c9-32b4726a6320/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-25T03:36:41,848 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efa42ab5-ac11-230a-67c9-32b4726a6320/nfs.dump.dir in system properties and HBase conf 2024-11-25T03:36:41,848 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efa42ab5-ac11-230a-67c9-32b4726a6320/java.io.tmpdir in system properties and HBase conf 2024-11-25T03:36:41,849 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efa42ab5-ac11-230a-67c9-32b4726a6320/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-25T03:36:41,849 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efa42ab5-ac11-230a-67c9-32b4726a6320/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-25T03:36:41,850 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efa42ab5-ac11-230a-67c9-32b4726a6320/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-25T03:36:42,839 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-25T03:36:42,938 INFO [Time-limited test {}] log.Log(170): Logging initialized @2828ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-25T03:36:43,041 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T03:36:43,115 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-25T03:36:43,142 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-25T03:36:43,142 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-25T03:36:43,144 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-25T03:36:43,158 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T03:36:43,161 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efa42ab5-ac11-230a-67c9-32b4726a6320/hadoop.log.dir/,AVAILABLE} 2024-11-25T03:36:43,162 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-25T03:36:43,401 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efa42ab5-ac11-230a-67c9-32b4726a6320/java.io.tmpdir/jetty-localhost-43635-hadoop-hdfs-3_4_1-tests_jar-_-any-4502660747289878472/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-25T03:36:43,409 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:43635} 2024-11-25T03:36:43,409 INFO [Time-limited test {}] server.Server(415): Started @3299ms 2024-11-25T03:36:43,845 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T03:36:43,853 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-25T03:36:43,856 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-25T03:36:43,856 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-25T03:36:43,856 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-25T03:36:43,857 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efa42ab5-ac11-230a-67c9-32b4726a6320/hadoop.log.dir/,AVAILABLE} 2024-11-25T03:36:43,858 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-25T03:36:43,985 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efa42ab5-ac11-230a-67c9-32b4726a6320/java.io.tmpdir/jetty-localhost-32935-hadoop-hdfs-3_4_1-tests_jar-_-any-14031796010275220664/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-25T03:36:43,986 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:32935} 2024-11-25T03:36:43,986 INFO [Time-limited test {}] server.Server(415): Started @3876ms 2024-11-25T03:36:44,049 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-25T03:36:44,579 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efa42ab5-ac11-230a-67c9-32b4726a6320/cluster_2bccc8e6-c56b-5b8b-fc38-b8aab8adb7e9/data/data2/current/BP-1682373972-172.17.0.2-1732505802553/current, will proceed with Du for space computation calculation, 2024-11-25T03:36:44,579 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efa42ab5-ac11-230a-67c9-32b4726a6320/cluster_2bccc8e6-c56b-5b8b-fc38-b8aab8adb7e9/data/data1/current/BP-1682373972-172.17.0.2-1732505802553/current, will proceed with Du for space computation calculation, 2024-11-25T03:36:44,647 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-25T03:36:44,713 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x96da39714caf57af with lease ID 0xe4f972e59d1d2d79: Processing first storage report for DS-c5c30d2e-abe5-42ac-be22-2444ce927292 from datanode DatanodeRegistration(127.0.0.1:37733, datanodeUuid=fd72fa4e-5abe-4ea1-a94d-ea69555b5c4c, infoPort=42889, infoSecurePort=0, ipcPort=36017, storageInfo=lv=-57;cid=testClusterID;nsid=1903152521;c=1732505802553) 2024-11-25T03:36:44,714 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x96da39714caf57af with lease ID 0xe4f972e59d1d2d79: from storage DS-c5c30d2e-abe5-42ac-be22-2444ce927292 node DatanodeRegistration(127.0.0.1:37733, datanodeUuid=fd72fa4e-5abe-4ea1-a94d-ea69555b5c4c, infoPort=42889, infoSecurePort=0, ipcPort=36017, storageInfo=lv=-57;cid=testClusterID;nsid=1903152521;c=1732505802553), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-25T03:36:44,715 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x96da39714caf57af with lease ID 0xe4f972e59d1d2d79: Processing first storage report for DS-05e3aab2-be2f-4394-9adc-62c82f812283 from datanode DatanodeRegistration(127.0.0.1:37733, datanodeUuid=fd72fa4e-5abe-4ea1-a94d-ea69555b5c4c, infoPort=42889, infoSecurePort=0, ipcPort=36017, storageInfo=lv=-57;cid=testClusterID;nsid=1903152521;c=1732505802553) 2024-11-25T03:36:44,715 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x96da39714caf57af with lease ID 0xe4f972e59d1d2d79: from storage DS-05e3aab2-be2f-4394-9adc-62c82f812283 node DatanodeRegistration(127.0.0.1:37733, datanodeUuid=fd72fa4e-5abe-4ea1-a94d-ea69555b5c4c, infoPort=42889, infoSecurePort=0, ipcPort=36017, storageInfo=lv=-57;cid=testClusterID;nsid=1903152521;c=1732505802553), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-25T03:36:44,802 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efa42ab5-ac11-230a-67c9-32b4726a6320 2024-11-25T03:36:44,882 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efa42ab5-ac11-230a-67c9-32b4726a6320/cluster_2bccc8e6-c56b-5b8b-fc38-b8aab8adb7e9/zookeeper_0, clientPort=63582, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efa42ab5-ac11-230a-67c9-32b4726a6320/cluster_2bccc8e6-c56b-5b8b-fc38-b8aab8adb7e9/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efa42ab5-ac11-230a-67c9-32b4726a6320/cluster_2bccc8e6-c56b-5b8b-fc38-b8aab8adb7e9/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-25T03:36:44,892 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63582 2024-11-25T03:36:44,907 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T03:36:44,911 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T03:36:45,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741825_1001 (size=7) 2024-11-25T03:36:45,615 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3 with version=8 2024-11-25T03:36:45,615 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/hbase-staging 2024-11-25T03:36:45,722 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-25T03:36:45,997 INFO [Time-limited test {}] client.ConnectionUtils(128): master/38a5d2077268:0 server-side Connection retries=45 2024-11-25T03:36:46,009 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T03:36:46,009 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T03:36:46,015 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T03:36:46,015 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T03:36:46,015 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T03:36:46,185 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-25T03:36:46,252 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-25T03:36:46,265 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-25T03:36:46,270 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T03:36:46,309 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 16739 (auto-detected) 2024-11-25T03:36:46,310 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-25T03:36:46,338 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42243 2024-11-25T03:36:46,363 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42243 connecting to ZooKeeper ensemble=127.0.0.1:63582 2024-11-25T03:36:46,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:422430x0, quorum=127.0.0.1:63582, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T03:36:46,403 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42243-0x1014d328acf0000 connected 2024-11-25T03:36:46,440 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T03:36:46,443 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T03:36:46,452 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42243-0x1014d328acf0000, quorum=127.0.0.1:63582, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T03:36:46,458 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3, hbase.cluster.distributed=false 2024-11-25T03:36:46,489 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42243-0x1014d328acf0000, quorum=127.0.0.1:63582, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T03:36:46,495 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42243 2024-11-25T03:36:46,496 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42243 2024-11-25T03:36:46,496 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42243 2024-11-25T03:36:46,497 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42243 2024-11-25T03:36:46,498 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42243 2024-11-25T03:36:46,633 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/38a5d2077268:0 server-side Connection retries=45 2024-11-25T03:36:46,635 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T03:36:46,635 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T03:36:46,635 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T03:36:46,636 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T03:36:46,636 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T03:36:46,639 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-25T03:36:46,641 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T03:36:46,642 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34881 2024-11-25T03:36:46,645 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34881 connecting to ZooKeeper ensemble=127.0.0.1:63582 2024-11-25T03:36:46,647 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T03:36:46,651 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T03:36:46,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:348810x0, quorum=127.0.0.1:63582, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T03:36:46,661 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34881-0x1014d328acf0001 connected 2024-11-25T03:36:46,661 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34881-0x1014d328acf0001, quorum=127.0.0.1:63582, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T03:36:46,666 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-25T03:36:46,680 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-25T03:36:46,683 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34881-0x1014d328acf0001, quorum=127.0.0.1:63582, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-25T03:36:46,689 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34881-0x1014d328acf0001, quorum=127.0.0.1:63582, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T03:36:46,692 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34881 2024-11-25T03:36:46,692 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34881 2024-11-25T03:36:46,696 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34881 2024-11-25T03:36:46,700 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34881 2024-11-25T03:36:46,701 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34881 2024-11-25T03:36:46,724 DEBUG [M:0;38a5d2077268:42243 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;38a5d2077268:42243 2024-11-25T03:36:46,725 INFO [master/38a5d2077268:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/38a5d2077268,42243,1732505805780 2024-11-25T03:36:46,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34881-0x1014d328acf0001, quorum=127.0.0.1:63582, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T03:36:46,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42243-0x1014d328acf0000, quorum=127.0.0.1:63582, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T03:36:46,735 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42243-0x1014d328acf0000, quorum=127.0.0.1:63582, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/38a5d2077268,42243,1732505805780 2024-11-25T03:36:46,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42243-0x1014d328acf0000, quorum=127.0.0.1:63582, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:36:46,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34881-0x1014d328acf0001, quorum=127.0.0.1:63582, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-25T03:36:46,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34881-0x1014d328acf0001, quorum=127.0.0.1:63582, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:36:46,760 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42243-0x1014d328acf0000, quorum=127.0.0.1:63582, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-25T03:36:46,762 INFO [master/38a5d2077268:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/38a5d2077268,42243,1732505805780 from backup master directory 2024-11-25T03:36:46,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34881-0x1014d328acf0001, quorum=127.0.0.1:63582, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T03:36:46,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42243-0x1014d328acf0000, quorum=127.0.0.1:63582, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/38a5d2077268,42243,1732505805780 2024-11-25T03:36:46,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42243-0x1014d328acf0000, quorum=127.0.0.1:63582, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T03:36:46,767 WARN [master/38a5d2077268:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T03:36:46,768 INFO [master/38a5d2077268:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=38a5d2077268,42243,1732505805780 2024-11-25T03:36:46,771 INFO [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-25T03:36:46,772 INFO [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-25T03:36:46,840 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/hbase.id] with ID: d693e68b-1234-49de-b9a6-51eec9db27cc 2024-11-25T03:36:46,841 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/.tmp/hbase.id 2024-11-25T03:36:46,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741826_1002 (size=42) 2024-11-25T03:36:46,854 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/.tmp/hbase.id]:[hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/hbase.id] 2024-11-25T03:36:46,906 INFO [master/38a5d2077268:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T03:36:46,911 INFO [master/38a5d2077268:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-25T03:36:46,933 INFO [master/38a5d2077268:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 20ms. 2024-11-25T03:36:46,937 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34881-0x1014d328acf0001, quorum=127.0.0.1:63582, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:36:46,937 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42243-0x1014d328acf0000, quorum=127.0.0.1:63582, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:36:46,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741827_1003 (size=196) 2024-11-25T03:36:46,973 INFO [master/38a5d2077268:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T03:36:46,976 INFO [master/38a5d2077268:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-25T03:36:46,993 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:36:46,998 INFO [master/38a5d2077268:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:36:47,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741828_1004 (size=1189) 2024-11-25T03:36:47,460 INFO [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/MasterData/data/master/store 2024-11-25T03:36:47,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741829_1005 (size=34) 2024-11-25T03:36:47,888 INFO [master/38a5d2077268:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-25T03:36:47,892 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:36:47,894 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-25T03:36:47,894 INFO [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:36:47,894 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:36:47,896 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-25T03:36:47,896 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:36:47,897 INFO [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:36:47,898 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732505807893Disabling compacts and flushes for region at 1732505807893Disabling writes for close at 1732505807896 (+3 ms)Writing region close event to WAL at 1732505807897 (+1 ms)Closed at 1732505807897 2024-11-25T03:36:47,901 WARN [master/38a5d2077268:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/MasterData/data/master/store/.initializing 2024-11-25T03:36:47,901 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/MasterData/WALs/38a5d2077268,42243,1732505805780 2024-11-25T03:36:47,913 INFO [master/38a5d2077268:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T03:36:47,931 INFO [master/38a5d2077268:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=38a5d2077268%2C42243%2C1732505805780, suffix=, logDir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/MasterData/WALs/38a5d2077268,42243,1732505805780, archiveDir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/MasterData/oldWALs, maxLogs=10 2024-11-25T03:36:47,956 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/MasterData/WALs/38a5d2077268,42243,1732505805780/38a5d2077268%2C42243%2C1732505805780.1732505807935, exclude list is [], retry=0 2024-11-25T03:36:47,973 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37733,DS-c5c30d2e-abe5-42ac-be22-2444ce927292,DISK] 2024-11-25T03:36:47,976 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-25T03:36:48,017 INFO [master/38a5d2077268:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/MasterData/WALs/38a5d2077268,42243,1732505805780/38a5d2077268%2C42243%2C1732505805780.1732505807935 2024-11-25T03:36:48,018 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42889:42889)] 2024-11-25T03:36:48,019 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:36:48,019 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:36:48,023 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:36:48,025 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:36:48,067 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:36:48,100 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-25T03:36:48,105 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:36:48,109 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:36:48,110 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:36:48,114 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-25T03:36:48,115 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:36:48,116 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:36:48,116 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:36:48,119 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-25T03:36:48,119 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:36:48,121 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:36:48,121 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:36:48,124 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-25T03:36:48,125 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:36:48,126 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:36:48,126 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:36:48,130 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:36:48,131 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:36:48,137 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:36:48,138 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:36:48,142 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T03:36:48,147 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:36:48,152 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:36:48,153 INFO [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60891786, jitterRate=-0.092641681432724}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T03:36:48,159 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732505808038Initializing all the Stores at 1732505808041 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732505808042 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732505808043 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732505808043Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732505808043Cleaning up temporary data from old regions at 1732505808138 (+95 ms)Region opened successfully at 1732505808158 (+20 ms) 2024-11-25T03:36:48,160 INFO [master/38a5d2077268:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-25T03:36:48,203 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b4fc07a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=38a5d2077268/172.17.0.2:0 2024-11-25T03:36:48,237 INFO [master/38a5d2077268:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-25T03:36:48,250 INFO [master/38a5d2077268:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-25T03:36:48,250 INFO [master/38a5d2077268:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-25T03:36:48,253 INFO [master/38a5d2077268:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-25T03:36:48,255 INFO [master/38a5d2077268:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-25T03:36:48,260 INFO [master/38a5d2077268:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-25T03:36:48,260 INFO [master/38a5d2077268:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-25T03:36:48,287 INFO [master/38a5d2077268:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-25T03:36:48,296 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42243-0x1014d328acf0000, quorum=127.0.0.1:63582, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-25T03:36:48,298 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-25T03:36:48,301 INFO [master/38a5d2077268:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-25T03:36:48,303 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42243-0x1014d328acf0000, quorum=127.0.0.1:63582, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-25T03:36:48,305 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-25T03:36:48,308 INFO [master/38a5d2077268:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-25T03:36:48,311 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42243-0x1014d328acf0000, quorum=127.0.0.1:63582, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-25T03:36:48,314 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-25T03:36:48,315 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42243-0x1014d328acf0000, quorum=127.0.0.1:63582, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-25T03:36:48,317 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-25T03:36:48,338 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42243-0x1014d328acf0000, quorum=127.0.0.1:63582, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-25T03:36:48,339 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-25T03:36:48,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34881-0x1014d328acf0001, quorum=127.0.0.1:63582, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T03:36:48,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42243-0x1014d328acf0000, quorum=127.0.0.1:63582, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T03:36:48,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34881-0x1014d328acf0001, quorum=127.0.0.1:63582, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:36:48,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42243-0x1014d328acf0000, quorum=127.0.0.1:63582, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:36:48,347 INFO [master/38a5d2077268:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=38a5d2077268,42243,1732505805780, sessionid=0x1014d328acf0000, setting cluster-up flag (Was=false) 2024-11-25T03:36:48,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42243-0x1014d328acf0000, quorum=127.0.0.1:63582, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:36:48,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34881-0x1014d328acf0001, quorum=127.0.0.1:63582, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:36:48,369 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-25T03:36:48,371 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=38a5d2077268,42243,1732505805780 2024-11-25T03:36:48,377 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42243-0x1014d328acf0000, quorum=127.0.0.1:63582, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:36:48,377 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34881-0x1014d328acf0001, quorum=127.0.0.1:63582, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:36:48,384 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-25T03:36:48,385 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=38a5d2077268,42243,1732505805780 2024-11-25T03:36:48,393 INFO [master/38a5d2077268:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-25T03:36:48,406 INFO [RS:0;38a5d2077268:34881 {}] regionserver.HRegionServer(746): ClusterId : d693e68b-1234-49de-b9a6-51eec9db27cc 2024-11-25T03:36:48,409 DEBUG [RS:0;38a5d2077268:34881 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-25T03:36:48,415 DEBUG [RS:0;38a5d2077268:34881 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-25T03:36:48,415 DEBUG [RS:0;38a5d2077268:34881 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-25T03:36:48,418 DEBUG [RS:0;38a5d2077268:34881 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-25T03:36:48,419 DEBUG [RS:0;38a5d2077268:34881 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dc2382, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=38a5d2077268/172.17.0.2:0 2024-11-25T03:36:48,442 DEBUG [RS:0;38a5d2077268:34881 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;38a5d2077268:34881 2024-11-25T03:36:48,446 INFO [RS:0;38a5d2077268:34881 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-25T03:36:48,447 INFO [RS:0;38a5d2077268:34881 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-25T03:36:48,447 DEBUG [RS:0;38a5d2077268:34881 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-25T03:36:48,450 INFO [RS:0;38a5d2077268:34881 {}] regionserver.HRegionServer(2659): reportForDuty to master=38a5d2077268,42243,1732505805780 with port=34881, startcode=1732505806588 2024-11-25T03:36:48,465 DEBUG [RS:0;38a5d2077268:34881 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-25T03:36:48,480 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-25T03:36:48,492 INFO [master/38a5d2077268:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-25T03:36:48,504 INFO [master/38a5d2077268:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-25T03:36:48,510 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34671, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-25T03:36:48,513 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 38a5d2077268,42243,1732505805780 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-25T03:36:48,517 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42243 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:48,521 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/38a5d2077268:0, corePoolSize=5, maxPoolSize=5 2024-11-25T03:36:48,522 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/38a5d2077268:0, corePoolSize=5, maxPoolSize=5 2024-11-25T03:36:48,522 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/38a5d2077268:0, corePoolSize=5, maxPoolSize=5 2024-11-25T03:36:48,522 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/38a5d2077268:0, corePoolSize=5, maxPoolSize=5 2024-11-25T03:36:48,522 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/38a5d2077268:0, corePoolSize=10, maxPoolSize=10 2024-11-25T03:36:48,523 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/38a5d2077268:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:36:48,523 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/38a5d2077268:0, corePoolSize=2, maxPoolSize=2 2024-11-25T03:36:48,523 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/38a5d2077268:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:36:48,529 INFO [master/38a5d2077268:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732505838529 2024-11-25T03:36:48,530 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T03:36:48,531 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-25T03:36:48,532 INFO [master/38a5d2077268:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-25T03:36:48,533 INFO [master/38a5d2077268:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-25T03:36:48,538 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:36:48,538 INFO [master/38a5d2077268:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-25T03:36:48,538 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-25T03:36:48,539 INFO [master/38a5d2077268:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-25T03:36:48,539 INFO [master/38a5d2077268:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-25T03:36:48,539 INFO [master/38a5d2077268:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-25T03:36:48,540 INFO [master/38a5d2077268:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-25T03:36:48,546 INFO [master/38a5d2077268:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-25T03:36:48,548 INFO [master/38a5d2077268:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-25T03:36:48,548 INFO [master/38a5d2077268:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-25T03:36:48,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741831_1007 (size=1321) 2024-11-25T03:36:48,551 INFO [master/38a5d2077268:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-25T03:36:48,552 INFO [master/38a5d2077268:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-25T03:36:48,553 DEBUG [RS:0;38a5d2077268:34881 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-25T03:36:48,553 WARN [RS:0;38a5d2077268:34881 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-25T03:36:48,556 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/38a5d2077268:0:becomeActiveMaster-HFileCleaner.large.0-1732505808553,5,FailOnTimeoutGroup] 2024-11-25T03:36:48,557 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/38a5d2077268:0:becomeActiveMaster-HFileCleaner.small.0-1732505808557,5,FailOnTimeoutGroup] 2024-11-25T03:36:48,557 INFO [master/38a5d2077268:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-25T03:36:48,558 INFO [master/38a5d2077268:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-25T03:36:48,559 INFO [master/38a5d2077268:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-25T03:36:48,559 INFO [master/38a5d2077268:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-25T03:36:48,654 INFO [RS:0;38a5d2077268:34881 {}] regionserver.HRegionServer(2659): reportForDuty to master=38a5d2077268,42243,1732505805780 with port=34881, startcode=1732505806588 2024-11-25T03:36:48,657 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42243 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 38a5d2077268,34881,1732505806588 2024-11-25T03:36:48,659 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42243 {}] master.ServerManager(517): Registering regionserver=38a5d2077268,34881,1732505806588 2024-11-25T03:36:48,668 DEBUG [RS:0;38a5d2077268:34881 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3 2024-11-25T03:36:48,668 DEBUG [RS:0;38a5d2077268:34881 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44505 2024-11-25T03:36:48,668 DEBUG [RS:0;38a5d2077268:34881 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-25T03:36:48,674 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42243-0x1014d328acf0000, quorum=127.0.0.1:63582, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T03:36:48,675 DEBUG [RS:0;38a5d2077268:34881 {}] zookeeper.ZKUtil(111): regionserver:34881-0x1014d328acf0001, quorum=127.0.0.1:63582, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/38a5d2077268,34881,1732505806588 2024-11-25T03:36:48,675 WARN [RS:0;38a5d2077268:34881 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T03:36:48,675 INFO [RS:0;38a5d2077268:34881 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:36:48,676 DEBUG [RS:0;38a5d2077268:34881 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/WALs/38a5d2077268,34881,1732505806588 2024-11-25T03:36:48,677 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [38a5d2077268,34881,1732505806588] 2024-11-25T03:36:48,710 INFO [RS:0;38a5d2077268:34881 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-25T03:36:48,729 INFO [RS:0;38a5d2077268:34881 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-25T03:36:48,735 INFO [RS:0;38a5d2077268:34881 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T03:36:48,735 INFO [RS:0;38a5d2077268:34881 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:36:48,737 INFO [RS:0;38a5d2077268:34881 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-25T03:36:48,745 INFO [RS:0;38a5d2077268:34881 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-25T03:36:48,747 INFO [RS:0;38a5d2077268:34881 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-25T03:36:48,748 DEBUG [RS:0;38a5d2077268:34881 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/38a5d2077268:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:36:48,748 DEBUG [RS:0;38a5d2077268:34881 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/38a5d2077268:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:36:48,748 DEBUG [RS:0;38a5d2077268:34881 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:36:48,749 DEBUG [RS:0;38a5d2077268:34881 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/38a5d2077268:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:36:48,749 DEBUG [RS:0;38a5d2077268:34881 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/38a5d2077268:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:36:48,749 DEBUG [RS:0;38a5d2077268:34881 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/38a5d2077268:0, corePoolSize=2, maxPoolSize=2 2024-11-25T03:36:48,749 DEBUG [RS:0;38a5d2077268:34881 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:36:48,750 DEBUG [RS:0;38a5d2077268:34881 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/38a5d2077268:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:36:48,750 DEBUG [RS:0;38a5d2077268:34881 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/38a5d2077268:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:36:48,750 DEBUG [RS:0;38a5d2077268:34881 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/38a5d2077268:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:36:48,750 DEBUG [RS:0;38a5d2077268:34881 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/38a5d2077268:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:36:48,751 DEBUG [RS:0;38a5d2077268:34881 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/38a5d2077268:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:36:48,751 DEBUG [RS:0;38a5d2077268:34881 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/38a5d2077268:0, corePoolSize=3, maxPoolSize=3 2024-11-25T03:36:48,751 DEBUG [RS:0;38a5d2077268:34881 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0, corePoolSize=3, maxPoolSize=3 2024-11-25T03:36:48,752 INFO [RS:0;38a5d2077268:34881 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T03:36:48,753 INFO [RS:0;38a5d2077268:34881 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T03:36:48,753 INFO [RS:0;38a5d2077268:34881 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:36:48,753 INFO [RS:0;38a5d2077268:34881 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-25T03:36:48,753 INFO [RS:0;38a5d2077268:34881 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-25T03:36:48,754 INFO [RS:0;38a5d2077268:34881 {}] hbase.ChoreService(168): Chore ScheduledChore name=38a5d2077268,34881,1732505806588-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T03:36:48,778 INFO [RS:0;38a5d2077268:34881 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-25T03:36:48,780 INFO [RS:0;38a5d2077268:34881 {}] hbase.ChoreService(168): Chore ScheduledChore name=38a5d2077268,34881,1732505806588-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:36:48,781 INFO [RS:0;38a5d2077268:34881 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:36:48,781 INFO [RS:0;38a5d2077268:34881 {}] regionserver.Replication(171): 38a5d2077268,34881,1732505806588 started 2024-11-25T03:36:48,800 INFO [RS:0;38a5d2077268:34881 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:36:48,800 INFO [RS:0;38a5d2077268:34881 {}] regionserver.HRegionServer(1482): Serving as 38a5d2077268,34881,1732505806588, RpcServer on 38a5d2077268/172.17.0.2:34881, sessionid=0x1014d328acf0001 2024-11-25T03:36:48,801 DEBUG [RS:0;38a5d2077268:34881 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-25T03:36:48,801 DEBUG [RS:0;38a5d2077268:34881 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 38a5d2077268,34881,1732505806588 2024-11-25T03:36:48,802 DEBUG [RS:0;38a5d2077268:34881 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '38a5d2077268,34881,1732505806588' 2024-11-25T03:36:48,802 DEBUG [RS:0;38a5d2077268:34881 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-25T03:36:48,803 DEBUG [RS:0;38a5d2077268:34881 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-25T03:36:48,804 DEBUG [RS:0;38a5d2077268:34881 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-25T03:36:48,804 DEBUG [RS:0;38a5d2077268:34881 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-25T03:36:48,804 DEBUG [RS:0;38a5d2077268:34881 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 38a5d2077268,34881,1732505806588 2024-11-25T03:36:48,804 DEBUG [RS:0;38a5d2077268:34881 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '38a5d2077268,34881,1732505806588' 2024-11-25T03:36:48,804 DEBUG [RS:0;38a5d2077268:34881 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-25T03:36:48,805 DEBUG [RS:0;38a5d2077268:34881 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-25T03:36:48,806 DEBUG [RS:0;38a5d2077268:34881 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-25T03:36:48,806 INFO [RS:0;38a5d2077268:34881 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-25T03:36:48,806 INFO [RS:0;38a5d2077268:34881 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-25T03:36:48,912 INFO [RS:0;38a5d2077268:34881 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T03:36:48,916 INFO [RS:0;38a5d2077268:34881 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=38a5d2077268%2C34881%2C1732505806588, suffix=, logDir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/WALs/38a5d2077268,34881,1732505806588, archiveDir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/oldWALs, maxLogs=32 2024-11-25T03:36:48,934 DEBUG [RS:0;38a5d2077268:34881 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/WALs/38a5d2077268,34881,1732505806588/38a5d2077268%2C34881%2C1732505806588.1732505808918, exclude list is [], retry=0 2024-11-25T03:36:48,939 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37733,DS-c5c30d2e-abe5-42ac-be22-2444ce927292,DISK] 2024-11-25T03:36:48,943 INFO [RS:0;38a5d2077268:34881 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/WALs/38a5d2077268,34881,1732505806588/38a5d2077268%2C34881%2C1732505806588.1732505808918 2024-11-25T03:36:48,943 DEBUG [RS:0;38a5d2077268:34881 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42889:42889)] 2024-11-25T03:36:48,954 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-25T03:36:48,954 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3 2024-11-25T03:36:48,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741833_1009 (size=32) 2024-11-25T03:36:49,368 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:36:49,371 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-25T03:36:49,374 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T03:36:49,374 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:36:49,375 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:36:49,376 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-25T03:36:49,379 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T03:36:49,379 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:36:49,380 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:36:49,380 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-25T03:36:49,383 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T03:36:49,383 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:36:49,384 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:36:49,384 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-25T03:36:49,387 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T03:36:49,387 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:36:49,388 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:36:49,388 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T03:36:49,390 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/hbase/meta/1588230740 2024-11-25T03:36:49,390 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/hbase/meta/1588230740 2024-11-25T03:36:49,393 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T03:36:49,393 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T03:36:49,394 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T03:36:49,397 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T03:36:49,401 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:36:49,402 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66194394, jitterRate=-0.013626664876937866}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T03:36:49,405 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732505809368Initializing all the Stores at 1732505809370 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732505809371 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732505809371Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732505809371Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732505809371Cleaning up temporary data from old regions at 1732505809393 (+22 ms)Region opened successfully at 1732505809405 (+12 ms) 2024-11-25T03:36:49,406 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-25T03:36:49,406 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-25T03:36:49,406 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-25T03:36:49,406 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-25T03:36:49,406 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-25T03:36:49,408 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-25T03:36:49,408 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732505809405Disabling compacts and flushes for region at 1732505809405Disabling writes for close at 1732505809406 (+1 ms)Writing region close event to WAL at 1732505809407 (+1 ms)Closed at 1732505809408 (+1 ms) 2024-11-25T03:36:49,412 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T03:36:49,413 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-25T03:36:49,422 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-25T03:36:49,434 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T03:36:49,437 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-25T03:36:49,590 DEBUG [38a5d2077268:42243 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-25T03:36:49,605 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=38a5d2077268,34881,1732505806588 2024-11-25T03:36:49,613 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 38a5d2077268,34881,1732505806588, state=OPENING 2024-11-25T03:36:49,618 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-25T03:36:49,620 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34881-0x1014d328acf0001, quorum=127.0.0.1:63582, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:36:49,620 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42243-0x1014d328acf0000, quorum=127.0.0.1:63582, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:36:49,621 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:36:49,621 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:36:49,622 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T03:36:49,624 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=38a5d2077268,34881,1732505806588}] 2024-11-25T03:36:49,803 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-25T03:36:49,807 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57951, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-25T03:36:49,821 INFO [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-25T03:36:49,822 INFO [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:36:49,823 INFO [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-25T03:36:49,827 INFO [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=38a5d2077268%2C34881%2C1732505806588.meta, suffix=.meta, logDir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/WALs/38a5d2077268,34881,1732505806588, archiveDir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/oldWALs, maxLogs=32 2024-11-25T03:36:49,846 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/WALs/38a5d2077268,34881,1732505806588/38a5d2077268%2C34881%2C1732505806588.meta.1732505809829.meta, exclude list is [], retry=0 2024-11-25T03:36:49,850 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37733,DS-c5c30d2e-abe5-42ac-be22-2444ce927292,DISK] 2024-11-25T03:36:49,854 INFO [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/WALs/38a5d2077268,34881,1732505806588/38a5d2077268%2C34881%2C1732505806588.meta.1732505809829.meta 2024-11-25T03:36:49,854 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42889:42889)] 2024-11-25T03:36:49,855 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:36:49,857 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-25T03:36:49,861 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-25T03:36:49,867 INFO [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-25T03:36:49,873 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-25T03:36:49,874 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:36:49,874 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-25T03:36:49,875 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-25T03:36:49,879 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-25T03:36:49,880 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T03:36:49,880 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:36:49,881 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:36:49,881 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-25T03:36:49,883 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T03:36:49,883 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:36:49,884 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:36:49,884 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-25T03:36:49,885 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T03:36:49,886 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:36:49,886 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:36:49,887 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-25T03:36:49,888 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T03:36:49,888 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:36:49,889 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:36:49,890 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T03:36:49,891 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/hbase/meta/1588230740 2024-11-25T03:36:49,894 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/hbase/meta/1588230740 2024-11-25T03:36:49,897 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T03:36:49,898 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T03:36:49,899 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T03:36:49,902 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T03:36:49,904 INFO [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63897741, jitterRate=-0.04784946143627167}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T03:36:49,904 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-25T03:36:49,906 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732505809875Writing region info on filesystem at 1732505809876 (+1 ms)Initializing all the Stores at 1732505809878 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732505809878Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732505809878Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732505809878Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732505809878Cleaning up temporary data from old regions at 1732505809898 (+20 ms)Running coprocessor post-open hooks at 1732505809904 (+6 ms)Region opened successfully at 1732505809906 (+2 ms) 2024-11-25T03:36:49,914 INFO [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732505809793 2024-11-25T03:36:49,926 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-25T03:36:49,927 INFO [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-25T03:36:49,928 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=38a5d2077268,34881,1732505806588 2024-11-25T03:36:49,931 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 38a5d2077268,34881,1732505806588, state=OPEN 2024-11-25T03:36:49,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42243-0x1014d328acf0000, quorum=127.0.0.1:63582, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:36:49,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34881-0x1014d328acf0001, quorum=127.0.0.1:63582, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:36:49,939 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:36:49,939 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:36:49,940 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:49,946 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-25T03:36:49,946 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=38a5d2077268,34881,1732505806588 in 316 msec 2024-11-25T03:36:49,953 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-25T03:36:49,953 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 527 msec 2024-11-25T03:36:49,954 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T03:36:49,955 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-25T03:36:49,979 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:36:49,981 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:36:50,010 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:36:50,013 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47205, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:36:50,042 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6120 sec 2024-11-25T03:36:50,042 INFO [master/38a5d2077268:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732505810042, completionTime=-1 2024-11-25T03:36:50,045 INFO [master/38a5d2077268:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-25T03:36:50,045 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-25T03:36:50,114 INFO [master/38a5d2077268:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-25T03:36:50,114 INFO [master/38a5d2077268:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732505870114 2024-11-25T03:36:50,114 INFO [master/38a5d2077268:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732505930114 2024-11-25T03:36:50,114 INFO [master/38a5d2077268:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 69 msec 2024-11-25T03:36:50,117 INFO [master/38a5d2077268:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=38a5d2077268,42243,1732505805780-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:36:50,118 INFO [master/38a5d2077268:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=38a5d2077268,42243,1732505805780-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:36:50,118 INFO [master/38a5d2077268:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=38a5d2077268,42243,1732505805780-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:36:50,120 INFO [master/38a5d2077268:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-38a5d2077268:42243, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:36:50,121 INFO [master/38a5d2077268:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-25T03:36:50,121 INFO [master/38a5d2077268:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-25T03:36:50,129 DEBUG [master/38a5d2077268:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-25T03:36:50,151 INFO [master/38a5d2077268:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.383sec 2024-11-25T03:36:50,152 INFO [master/38a5d2077268:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-25T03:36:50,153 INFO [master/38a5d2077268:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-25T03:36:50,154 INFO [master/38a5d2077268:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-25T03:36:50,155 INFO [master/38a5d2077268:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-25T03:36:50,155 INFO [master/38a5d2077268:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-25T03:36:50,156 INFO [master/38a5d2077268:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=38a5d2077268,42243,1732505805780-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T03:36:50,156 INFO [master/38a5d2077268:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=38a5d2077268,42243,1732505805780-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-25T03:36:50,165 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-25T03:36:50,166 INFO [master/38a5d2077268:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-25T03:36:50,167 INFO [master/38a5d2077268:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=38a5d2077268,42243,1732505805780-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:36:50,220 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45544f78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:36:50,222 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-25T03:36:50,222 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-25T03:36:50,226 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:36:50,229 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:36:50,237 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:36:50,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:36:50,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:36:50,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d7aa7eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:36:50,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:36:50,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:36:50,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:36:50,250 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50748, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:36:50,253 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2de67725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:36:50,253 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:36:50,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:36:50,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:36:50,266 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49788, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:36:50,286 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=38a5d2077268,42243,1732505805780 2024-11-25T03:36:50,305 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=193, ProcessCount=11, AvailableMemoryMB=6630 2024-11-25T03:36:50,318 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T03:36:50,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 38a5d2077268,42243,1732505805780 2024-11-25T03:36:50,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5b72eb06 2024-11-25T03:36:50,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T03:36:50,331 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50764, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T03:36:50,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T03:36:50,339 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T03:36:50,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-25T03:36:50,352 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T03:36:50,354 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-25T03:36:50,354 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:36:50,357 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T03:36:50,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T03:36:50,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741835_1011 (size=959) 2024-11-25T03:36:50,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T03:36:50,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T03:36:50,775 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3 2024-11-25T03:36:50,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741836_1012 (size=53) 2024-11-25T03:36:50,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T03:36:51,186 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:36:51,186 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ee186277170c2bdfb070a7c209286857, disabling compactions & flushes 2024-11-25T03:36:51,186 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:36:51,186 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:36:51,186 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. after waiting 0 ms 2024-11-25T03:36:51,186 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:36:51,186 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:36:51,186 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ee186277170c2bdfb070a7c209286857: Waiting for close lock at 1732505811186Disabling compacts and flushes for region at 1732505811186Disabling writes for close at 1732505811186Writing region close event to WAL at 1732505811186Closed at 1732505811186 2024-11-25T03:36:51,189 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T03:36:51,194 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732505811189"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732505811189"}]},"ts":"1732505811189"} 2024-11-25T03:36:51,199 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T03:36:51,201 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T03:36:51,203 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732505811201"}]},"ts":"1732505811201"} 2024-11-25T03:36:51,208 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-25T03:36:51,210 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee186277170c2bdfb070a7c209286857, ASSIGN}] 2024-11-25T03:36:51,213 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee186277170c2bdfb070a7c209286857, ASSIGN 2024-11-25T03:36:51,215 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee186277170c2bdfb070a7c209286857, ASSIGN; state=OFFLINE, location=38a5d2077268,34881,1732505806588; forceNewPlan=false, retain=false 2024-11-25T03:36:51,366 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ee186277170c2bdfb070a7c209286857, regionState=OPENING, regionLocation=38a5d2077268,34881,1732505806588 2024-11-25T03:36:51,371 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee186277170c2bdfb070a7c209286857, ASSIGN because future has completed 2024-11-25T03:36:51,372 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588}] 2024-11-25T03:36:51,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T03:36:51,534 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:36:51,534 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:36:51,535 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ee186277170c2bdfb070a7c209286857 2024-11-25T03:36:51,535 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:36:51,535 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for ee186277170c2bdfb070a7c209286857 2024-11-25T03:36:51,535 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for ee186277170c2bdfb070a7c209286857 2024-11-25T03:36:51,538 INFO [StoreOpener-ee186277170c2bdfb070a7c209286857-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ee186277170c2bdfb070a7c209286857 2024-11-25T03:36:51,541 INFO [StoreOpener-ee186277170c2bdfb070a7c209286857-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee186277170c2bdfb070a7c209286857 columnFamilyName A 2024-11-25T03:36:51,541 DEBUG [StoreOpener-ee186277170c2bdfb070a7c209286857-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:36:51,543 INFO [StoreOpener-ee186277170c2bdfb070a7c209286857-1 {}] regionserver.HStore(327): Store=ee186277170c2bdfb070a7c209286857/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:36:51,543 INFO [StoreOpener-ee186277170c2bdfb070a7c209286857-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ee186277170c2bdfb070a7c209286857 2024-11-25T03:36:51,545 INFO [StoreOpener-ee186277170c2bdfb070a7c209286857-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee186277170c2bdfb070a7c209286857 columnFamilyName B 2024-11-25T03:36:51,545 DEBUG [StoreOpener-ee186277170c2bdfb070a7c209286857-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:36:51,546 INFO [StoreOpener-ee186277170c2bdfb070a7c209286857-1 {}] regionserver.HStore(327): Store=ee186277170c2bdfb070a7c209286857/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:36:51,547 INFO [StoreOpener-ee186277170c2bdfb070a7c209286857-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ee186277170c2bdfb070a7c209286857 2024-11-25T03:36:51,549 INFO [StoreOpener-ee186277170c2bdfb070a7c209286857-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee186277170c2bdfb070a7c209286857 columnFamilyName C 2024-11-25T03:36:51,549 DEBUG [StoreOpener-ee186277170c2bdfb070a7c209286857-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:36:51,550 INFO [StoreOpener-ee186277170c2bdfb070a7c209286857-1 {}] regionserver.HStore(327): Store=ee186277170c2bdfb070a7c209286857/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:36:51,550 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for ee186277170c2bdfb070a7c209286857 2024-11-25T03:36:51,552 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857 2024-11-25T03:36:51,552 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857 2024-11-25T03:36:51,554 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for ee186277170c2bdfb070a7c209286857 2024-11-25T03:36:51,554 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for ee186277170c2bdfb070a7c209286857 2024-11-25T03:36:51,555 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T03:36:51,558 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for ee186277170c2bdfb070a7c209286857 2024-11-25T03:36:51,562 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:36:51,563 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened ee186277170c2bdfb070a7c209286857; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74496464, jitterRate=0.11008381843566895}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T03:36:51,563 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ee186277170c2bdfb070a7c209286857 2024-11-25T03:36:51,564 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for ee186277170c2bdfb070a7c209286857: Running coprocessor pre-open hook at 1732505811535Writing region info on filesystem at 1732505811535Initializing all the Stores at 1732505811537 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732505811537Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732505811538 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732505811538Cleaning up temporary data from old regions at 1732505811554 (+16 ms)Running coprocessor post-open hooks at 1732505811563 (+9 ms)Region opened successfully at 1732505811564 (+1 ms) 2024-11-25T03:36:51,566 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., pid=6, masterSystemTime=1732505811526 2024-11-25T03:36:51,570 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:36:51,570 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:36:51,571 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ee186277170c2bdfb070a7c209286857, regionState=OPEN, openSeqNum=2, regionLocation=38a5d2077268,34881,1732505806588 2024-11-25T03:36:51,576 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 because future has completed 2024-11-25T03:36:51,584 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-25T03:36:51,584 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 in 209 msec 2024-11-25T03:36:51,588 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-25T03:36:51,588 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee186277170c2bdfb070a7c209286857, ASSIGN in 374 msec 2024-11-25T03:36:51,589 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T03:36:51,589 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732505811589"}]},"ts":"1732505811589"} 2024-11-25T03:36:51,593 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-25T03:36:51,594 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T03:36:51,597 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2520 sec 2024-11-25T03:36:52,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T03:36:52,510 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:36:52,516 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19b2c994, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:36:52,516 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:36:52,516 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:36:52,518 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:36:52,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:36:52,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:36:52,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d34895b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:36:52,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:36:52,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:36:52,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:36:52,521 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43310, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:36:52,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50806732, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:36:52,524 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:36:52,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:36:52,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:36:52,529 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60536, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:36:52,532 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T03:36:52,534 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 38a5d2077268,42243,1732505805780 2024-11-25T03:36:52,534 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3d4a6a92 2024-11-25T03:36:52,536 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T03:36:52,540 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43326, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T03:36:52,545 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a52ffa9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:36:52,545 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:36:52,545 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:36:52,547 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:36:52,548 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:36:52,548 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:36:52,548 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e11900a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:36:52,548 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:36:52,549 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:36:52,549 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:36:52,551 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43350, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:36:52,552 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ca0b5f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:36:52,553 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51f14cda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:36:52,553 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:36:52,553 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:36:52,555 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:36:52,555 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:36:52,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:36:52,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@387dfdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:36:52,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:36:52,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:36:52,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:36:52,558 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43372, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:36:52,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25175df8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:36:52,562 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4201ea29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:36:52,562 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:36:52,562 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:36:52,564 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:36:52,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:36:52,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:36:52,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f4e6a07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:36:52,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:36:52,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:36:52,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:36:52,567 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43382, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:36:52,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b17c100, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:36:52,569 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45cc41cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:36:52,570 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:36:52,570 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:36:52,571 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:36:52,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:36:52,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:36:52,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e3687d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:36:52,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:36:52,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:36:52,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:36:52,575 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43402, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:36:52,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10112b3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:36:52,576 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2900b22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:36:52,576 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:36:52,577 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:36:52,578 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:36:52,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:36:52,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:36:52,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c3f9496, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:36:52,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:36:52,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:36:52,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:36:52,580 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43424, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:36:52,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a213a8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:36:52,585 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d39686f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:36:52,585 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:36:52,586 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:36:52,587 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:36:52,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:36:52,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:36:52,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b71435d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:36:52,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:36:52,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:36:52,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:36:52,590 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43448, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:36:52,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f6d26ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:36:52,592 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6963d59b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:36:52,592 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:36:52,592 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:36:52,593 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:36:52,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:36:52,594 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:36:52,594 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79aeff1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:36:52,594 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:36:52,594 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:36:52,594 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:36:52,595 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:36:52,596 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64994b9f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:36:52,598 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b9a31d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:36:52,598 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:36:52,598 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:36:52,599 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:36:52,599 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:36:52,599 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:36:52,599 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f551be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:36:52,599 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:36:52,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:36:52,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:36:52,601 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43484, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:36:52,602 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@189704a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:36:52,603 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0b495b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:36:52,603 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:36:52,603 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:36:52,604 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:36:52,604 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:36:52,604 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:36:52,604 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7299e0d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:36:52,604 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:36:52,604 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:36:52,605 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:36:52,606 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43492, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:36:52,607 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39568e1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:36:52,611 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:36:52,611 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:36:52,613 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:36:52,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:36:52,613 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:36:52,613 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:36:52,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:36:52,614 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:36:52,615 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60560, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:36:52,616 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60552, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:36:52,616 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:36:52,616 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:36:52,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:36:52,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:36:52,618 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:36:52,620 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60572, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:36:52,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2] 2024-11-25T03:36:52,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2] 2024-11-25T03:36:52,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2] 2024-11-25T03:36:52,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2] 2024-11-25T03:36:52,626 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:36:52,629 DEBUG [Thread-149 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:36:52,630 DEBUG [Thread-149 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:36:52,631 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:36:52,632 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60576, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:36:52,633 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:36:52,635 DEBUG [Thread-143 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:36:52,635 DEBUG [Thread-143 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:36:52,638 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60588, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:36:52,641 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:36:52,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-25T03:36:52,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2] 2024-11-25T03:36:52,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:36:52,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:36:52,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-25T03:36:52,645 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:36:52,647 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:36:52,641 DEBUG [Thread-149 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2] 2024-11-25T03:36:52,647 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:36:52,648 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:36:52,649 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60596, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:36:52,650 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60590, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:36:52,651 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:36:52,653 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:36:52,655 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:36:52,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2] 2024-11-25T03:36:52,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:36:52,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:36:52,660 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60602, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:36:52,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2] 2024-11-25T03:36:52,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2] 2024-11-25T03:36:52,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on ee186277170c2bdfb070a7c209286857 2024-11-25T03:36:52,728 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:36:52,733 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee186277170c2bdfb070a7c209286857 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-25T03:36:52,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-25T03:36:52,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-25T03:36:52,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:36:52,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. as already flushing 2024-11-25T03:36:52,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:36:52,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:36:52,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:36:52,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:36:52,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/7298417c4ab745ddb2f32b0c8a12c586 is 50, key is test_row_0/A:col10/1732505812713/Put/seqid=0 2024-11-25T03:36:52,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:52,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741837_1013 (size=14341) 2024-11-25T03:36:52,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:52,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505872930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:52,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505872906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:52,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:52,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:52,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:52,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:52,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:52,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:52,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:52,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505872941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:52,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:52,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:52,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:52,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-25T03:36:52,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:52,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505872948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:52,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:52,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:52,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:52,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:52,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505872950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:52,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:52,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:52,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:53,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-25T03:36:53,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:36:53,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. as already flushing 2024-11-25T03:36:53,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:36:53,039 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:36:53,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:36:53,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:36:53,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:53,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505873058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:53,070 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:53,070 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:53,070 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:53,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:53,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505873070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:53,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:53,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:53,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:53,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:53,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:53,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505873071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:53,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:53,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:53,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:53,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505873070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:53,075 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:53,075 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:53,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:53,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:53,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505873078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:53,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:53,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:53,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:53,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-25T03:36:53,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:36:53,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. as already flushing 2024-11-25T03:36:53,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:36:53,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:36:53,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:36:53,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:36:53,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-25T03:36:53,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:53,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505873278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:53,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:53,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:53,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:53,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:53,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:53,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505873279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:53,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505873279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:53,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:53,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:53,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:53,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:53,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:53,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:53,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:53,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:53,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505873289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:53,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505873289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:53,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:53,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:53,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:53,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:53,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:53,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:53,349 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/7298417c4ab745ddb2f32b0c8a12c586 2024-11-25T03:36:53,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-25T03:36:53,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:36:53,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. as already flushing 2024-11-25T03:36:53,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:36:53,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:36:53,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:36:53,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:36:53,513 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/6783223d439b44ab87c1b0b7c6149edc is 50, key is test_row_0/B:col10/1732505812713/Put/seqid=0 2024-11-25T03:36:53,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-25T03:36:53,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:36:53,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. as already flushing 2024-11-25T03:36:53,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:36:53,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:36:53,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:36:53,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:36:53,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741838_1014 (size=12001) 2024-11-25T03:36:53,574 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/6783223d439b44ab87c1b0b7c6149edc 2024-11-25T03:36:53,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:53,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:53,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505873589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:53,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505873589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:53,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:53,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:53,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:53,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:53,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:53,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:53,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:53,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505873591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:53,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:53,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:53,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:53,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:53,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505873608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:53,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:53,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:53,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:53,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:53,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505873608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:53,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:53,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:53,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:53,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/8140caf5e34d4693bd1f103cacf50b6e is 50, key is test_row_0/C:col10/1732505812713/Put/seqid=0 2024-11-25T03:36:53,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-25T03:36:53,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:36:53,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. as already flushing 2024-11-25T03:36:53,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:36:53,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:36:53,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:36:53,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741839_1015 (size=12001) 2024-11-25T03:36:53,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:36:53,701 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/8140caf5e34d4693bd1f103cacf50b6e 2024-11-25T03:36:53,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/7298417c4ab745ddb2f32b0c8a12c586 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/7298417c4ab745ddb2f32b0c8a12c586 2024-11-25T03:36:53,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/7298417c4ab745ddb2f32b0c8a12c586, entries=200, sequenceid=14, filesize=14.0 K 2024-11-25T03:36:53,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/6783223d439b44ab87c1b0b7c6149edc as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/6783223d439b44ab87c1b0b7c6149edc 2024-11-25T03:36:53,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-25T03:36:53,789 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/6783223d439b44ab87c1b0b7c6149edc, entries=150, sequenceid=14, filesize=11.7 K 2024-11-25T03:36:53,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/8140caf5e34d4693bd1f103cacf50b6e as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/8140caf5e34d4693bd1f103cacf50b6e 2024-11-25T03:36:53,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/8140caf5e34d4693bd1f103cacf50b6e, entries=150, sequenceid=14, filesize=11.7 K 2024-11-25T03:36:53,822 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for ee186277170c2bdfb070a7c209286857 in 1087ms, sequenceid=14, compaction requested=false 2024-11-25T03:36:53,824 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-25T03:36:53,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:36:53,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-25T03:36:53,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:36:53,858 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing ee186277170c2bdfb070a7c209286857 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-25T03:36:53,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/9cc336ed6808431d803a3116580beb05 is 50, key is test_row_0/A:col10/1732505812924/Put/seqid=0 2024-11-25T03:36:53,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741840_1016 (size=12001) 2024-11-25T03:36:54,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on ee186277170c2bdfb070a7c209286857 2024-11-25T03:36:54,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. as already flushing 2024-11-25T03:36:54,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:54,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505874151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:54,162 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:54,162 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:54,162 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:54,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:54,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505874152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:54,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:54,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:54,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:54,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:54,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505874155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:54,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:54,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:54,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:54,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:54,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505874159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:54,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:54,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:54,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:54,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:54,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505874158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:54,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:54,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:54,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:54,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:54,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505874268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:54,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:54,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:54,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:54,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:54,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505874268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:54,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:54,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:54,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:54,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:54,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:54,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505874278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:54,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505874278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:54,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:54,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:54,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:54,285 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:54,287 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:54,287 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:54,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:54,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505874288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:54,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:54,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:54,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:54,313 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/9cc336ed6808431d803a3116580beb05 2024-11-25T03:36:54,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/b025414f7057477c8fc36a619ccafe9d is 50, key is test_row_0/B:col10/1732505812924/Put/seqid=0 2024-11-25T03:36:54,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741841_1017 (size=12001) 2024-11-25T03:36:54,409 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/b025414f7057477c8fc36a619ccafe9d 2024-11-25T03:36:54,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/3eed32b972b6499599061d1e213b8b08 is 50, key is test_row_0/C:col10/1732505812924/Put/seqid=0 2024-11-25T03:36:54,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:54,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:54,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505874478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:54,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505874478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:54,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:54,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:54,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:54,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:54,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:54,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:54,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741842_1018 (size=12001) 2024-11-25T03:36:54,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:54,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505874498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:54,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:54,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:54,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:54,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:54,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505874498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:54,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:54,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:54,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:54,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:54,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505874507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:54,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:54,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:54,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:54,701 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-25T03:36:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-25T03:36:54,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:54,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505874798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:54,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:54,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:54,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:54,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:54,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505874798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:54,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:54,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:54,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:54,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:54,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505874807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:54,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:54,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:54,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:54,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:54,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505874818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:54,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:54,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:54,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:54,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:54,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505874819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:54,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:54,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:54,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:54,896 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/3eed32b972b6499599061d1e213b8b08 2024-11-25T03:36:54,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/9cc336ed6808431d803a3116580beb05 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/9cc336ed6808431d803a3116580beb05 2024-11-25T03:36:54,937 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/9cc336ed6808431d803a3116580beb05, entries=150, sequenceid=38, filesize=11.7 K 2024-11-25T03:36:54,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/b025414f7057477c8fc36a619ccafe9d as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/b025414f7057477c8fc36a619ccafe9d 2024-11-25T03:36:54,959 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/b025414f7057477c8fc36a619ccafe9d, entries=150, sequenceid=38, filesize=11.7 K 2024-11-25T03:36:54,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/3eed32b972b6499599061d1e213b8b08 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/3eed32b972b6499599061d1e213b8b08 2024-11-25T03:36:54,984 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/3eed32b972b6499599061d1e213b8b08, entries=150, sequenceid=38, filesize=11.7 K 2024-11-25T03:36:54,987 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=73.80 KB/75570 for ee186277170c2bdfb070a7c209286857 in 1129ms, sequenceid=38, compaction requested=false 2024-11-25T03:36:54,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:36:54,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:36:54,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-25T03:36:54,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-25T03:36:54,999 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-25T03:36:54,999 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3380 sec 2024-11-25T03:36:55,005 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.3680 sec 2024-11-25T03:36:55,317 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:36:55,317 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee186277170c2bdfb070a7c209286857 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-25T03:36:55,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on ee186277170c2bdfb070a7c209286857 2024-11-25T03:36:55,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/ab0a6e86e32e4b8db351200d8bd42d47 is 50, key is test_row_0/A:col10/1732505815311/Put/seqid=0 2024-11-25T03:36:55,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741843_1019 (size=14341) 2024-11-25T03:36:55,368 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/ab0a6e86e32e4b8db351200d8bd42d47 2024-11-25T03:36:55,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:55,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505875379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:55,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:55,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:55,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:55,397 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T03:36:55,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/a144230e0cee4cfba4b71b1755744ee1 is 50, key is test_row_0/B:col10/1732505815311/Put/seqid=0 2024-11-25T03:36:55,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:55,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505875387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:55,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:55,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505875390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:55,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:55,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505875391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:55,403 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:55,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:55,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:55,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:55,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505875392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:55,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:55,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:55,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:55,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:55,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:55,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:55,406 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:55,406 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:55,406 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:55,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741844_1020 (size=12001) 2024-11-25T03:36:55,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:55,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505875499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:55,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:55,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:55,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:55,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:55,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505875508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:55,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:55,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505875508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:55,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:55,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:55,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:55,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:55,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:55,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:55,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:55,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505875513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:55,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:55,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:55,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:55,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:55,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505875513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:55,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:55,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:55,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:55,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:55,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:55,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505875720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:55,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505875719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:55,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:55,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:55,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:55,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:55,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:55,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:55,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:55,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505875720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:55,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:55,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:55,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:55,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:55,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505875727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:55,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:55,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505875728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:55,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:55,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:55,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:55,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:55,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:55,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:55,875 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/a144230e0cee4cfba4b71b1755744ee1 2024-11-25T03:36:55,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/f911e024602b44f391a1abdfb25e2900 is 50, key is test_row_0/C:col10/1732505815311/Put/seqid=0 2024-11-25T03:36:55,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741845_1021 (size=12001) 2024-11-25T03:36:55,941 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/f911e024602b44f391a1abdfb25e2900 2024-11-25T03:36:55,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/ab0a6e86e32e4b8db351200d8bd42d47 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/ab0a6e86e32e4b8db351200d8bd42d47 2024-11-25T03:36:55,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/ab0a6e86e32e4b8db351200d8bd42d47, entries=200, sequenceid=53, filesize=14.0 K 2024-11-25T03:36:55,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/a144230e0cee4cfba4b71b1755744ee1 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/a144230e0cee4cfba4b71b1755744ee1 2024-11-25T03:36:55,990 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/a144230e0cee4cfba4b71b1755744ee1, entries=150, sequenceid=53, filesize=11.7 K 2024-11-25T03:36:55,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/f911e024602b44f391a1abdfb25e2900 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/f911e024602b44f391a1abdfb25e2900 2024-11-25T03:36:56,012 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/f911e024602b44f391a1abdfb25e2900, entries=150, sequenceid=53, filesize=11.7 K 2024-11-25T03:36:56,014 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for ee186277170c2bdfb070a7c209286857 in 697ms, sequenceid=53, compaction requested=true 2024-11-25T03:36:56,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:36:56,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee186277170c2bdfb070a7c209286857:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:36:56,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:36:56,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on ee186277170c2bdfb070a7c209286857 2024-11-25T03:36:56,035 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:36:56,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee186277170c2bdfb070a7c209286857:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:36:56,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:36:56,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee186277170c2bdfb070a7c209286857:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:36:56,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:36:56,040 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:36:56,040 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:36:56,040 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee186277170c2bdfb070a7c209286857 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-25T03:36:56,042 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): ee186277170c2bdfb070a7c209286857/B is initiating minor compaction (all files) 2024-11-25T03:36:56,043 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee186277170c2bdfb070a7c209286857/B in TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:36:56,043 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/6783223d439b44ab87c1b0b7c6149edc, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/b025414f7057477c8fc36a619ccafe9d, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/a144230e0cee4cfba4b71b1755744ee1] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp, totalSize=35.2 K 2024-11-25T03:36:56,046 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 6783223d439b44ab87c1b0b7c6149edc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732505812669 2024-11-25T03:36:56,048 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting b025414f7057477c8fc36a619ccafe9d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732505812924 2024-11-25T03:36:56,050 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting a144230e0cee4cfba4b71b1755744ee1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732505814141 2024-11-25T03:36:56,057 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:36:56,063 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:36:56,063 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): ee186277170c2bdfb070a7c209286857/A is initiating minor compaction (all files) 2024-11-25T03:36:56,064 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee186277170c2bdfb070a7c209286857/A in TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:36:56,064 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/7298417c4ab745ddb2f32b0c8a12c586, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/9cc336ed6808431d803a3116580beb05, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/ab0a6e86e32e4b8db351200d8bd42d47] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp, totalSize=39.7 K 2024-11-25T03:36:56,065 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7298417c4ab745ddb2f32b0c8a12c586, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732505812669 2024-11-25T03:36:56,066 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9cc336ed6808431d803a3116580beb05, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732505812924 2024-11-25T03:36:56,067 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab0a6e86e32e4b8db351200d8bd42d47, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732505814141 2024-11-25T03:36:56,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/f46bc099f04745e3930cd831c7a8791d is 50, key is test_row_0/A:col10/1732505815384/Put/seqid=0 2024-11-25T03:36:56,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:56,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505876062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:56,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:56,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:56,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:56,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:56,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505876063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:56,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:56,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:56,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:56,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:56,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505876074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:56,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:56,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:56,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:56,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:56,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505876077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:56,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:56,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:56,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:56,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:56,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505876081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:56,092 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:56,092 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:56,092 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:56,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741846_1022 (size=12001) 2024-11-25T03:36:56,111 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/f46bc099f04745e3930cd831c7a8791d 2024-11-25T03:36:56,127 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee186277170c2bdfb070a7c209286857#B#compaction#10 average throughput is 0.24 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:36:56,128 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee186277170c2bdfb070a7c209286857#A#compaction#11 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:36:56,128 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/7dfc7fe3067d44e4a3c122615f020c60 is 50, key is test_row_0/B:col10/1732505815311/Put/seqid=0 2024-11-25T03:36:56,132 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/4ccec1bd1585433aa572d844cca23ddb is 50, key is test_row_0/A:col10/1732505815311/Put/seqid=0 2024-11-25T03:36:56,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741847_1023 (size=12104) 2024-11-25T03:36:56,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/7a1df036c47b402db6d29ea4340f178f is 50, key is test_row_0/B:col10/1732505815384/Put/seqid=0 2024-11-25T03:36:56,177 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/7dfc7fe3067d44e4a3c122615f020c60 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/7dfc7fe3067d44e4a3c122615f020c60 2024-11-25T03:36:56,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:56,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505876187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:56,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741848_1024 (size=12104) 2024-11-25T03:36:56,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:56,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:56,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:56,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:56,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505876198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:56,207 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee186277170c2bdfb070a7c209286857/B of ee186277170c2bdfb070a7c209286857 into 7dfc7fe3067d44e4a3c122615f020c60(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:36:56,207 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:36:56,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:56,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:56,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:56,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:56,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505876199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:56,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:56,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505876199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:56,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:56,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:56,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:56,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:56,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:56,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:56,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:56,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505876199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:56,210 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., storeName=ee186277170c2bdfb070a7c209286857/B, priority=13, startTime=1732505816032; duration=0sec 2024-11-25T03:36:56,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:56,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:56,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:56,211 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:36:56,211 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee186277170c2bdfb070a7c209286857:B 2024-11-25T03:36:56,211 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:36:56,219 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:36:56,219 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): ee186277170c2bdfb070a7c209286857/C is initiating minor compaction (all files) 2024-11-25T03:36:56,219 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee186277170c2bdfb070a7c209286857/C in TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:36:56,220 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/8140caf5e34d4693bd1f103cacf50b6e, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/3eed32b972b6499599061d1e213b8b08, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/f911e024602b44f391a1abdfb25e2900] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp, totalSize=35.2 K 2024-11-25T03:36:56,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741849_1025 (size=12001) 2024-11-25T03:36:56,222 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 8140caf5e34d4693bd1f103cacf50b6e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732505812669 2024-11-25T03:36:56,222 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/7a1df036c47b402db6d29ea4340f178f 2024-11-25T03:36:56,223 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 3eed32b972b6499599061d1e213b8b08, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732505812924 2024-11-25T03:36:56,225 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting f911e024602b44f391a1abdfb25e2900, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732505814141 2024-11-25T03:36:56,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/59c9fa86e7394eef8e0768ba0efb37d7 is 50, key is test_row_0/C:col10/1732505815384/Put/seqid=0 2024-11-25T03:36:56,248 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-25T03:36:56,249 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-25T03:36:56,251 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-25T03:36:56,251 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-25T03:36:56,252 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-25T03:36:56,252 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-25T03:36:56,254 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-25T03:36:56,254 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-25T03:36:56,274 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee186277170c2bdfb070a7c209286857#C#compaction#14 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:36:56,275 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/ae61d993d0f044b79e81ee41db2933cb is 50, key is test_row_0/C:col10/1732505815311/Put/seqid=0 2024-11-25T03:36:56,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741850_1026 (size=12001) 2024-11-25T03:36:56,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741851_1027 (size=12104) 2024-11-25T03:36:56,341 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/ae61d993d0f044b79e81ee41db2933cb as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/ae61d993d0f044b79e81ee41db2933cb 2024-11-25T03:36:56,355 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee186277170c2bdfb070a7c209286857/C of ee186277170c2bdfb070a7c209286857 into ae61d993d0f044b79e81ee41db2933cb(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:36:56,356 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:36:56,356 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., storeName=ee186277170c2bdfb070a7c209286857/C, priority=13, startTime=1732505816039; duration=0sec 2024-11-25T03:36:56,356 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:36:56,356 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee186277170c2bdfb070a7c209286857:C 2024-11-25T03:36:56,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:56,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505876398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:56,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:56,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:56,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:56,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:56,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:56,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505876419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:56,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505876419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:56,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:56,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:56,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:56,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:56,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:56,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:56,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:56,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505876419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:56,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:56,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505876427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:56,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:56,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:56,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:56,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:56,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:56,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:56,605 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/4ccec1bd1585433aa572d844cca23ddb as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/4ccec1bd1585433aa572d844cca23ddb 2024-11-25T03:36:56,621 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee186277170c2bdfb070a7c209286857/A of ee186277170c2bdfb070a7c209286857 into 4ccec1bd1585433aa572d844cca23ddb(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:36:56,621 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:36:56,623 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., storeName=ee186277170c2bdfb070a7c209286857/A, priority=13, startTime=1732505816016; duration=0sec 2024-11-25T03:36:56,623 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:36:56,623 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee186277170c2bdfb070a7c209286857:A 2024-11-25T03:36:56,703 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/59c9fa86e7394eef8e0768ba0efb37d7 2024-11-25T03:36:56,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:56,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505876709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:56,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:56,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:56,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:56,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/f46bc099f04745e3930cd831c7a8791d as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/f46bc099f04745e3930cd831c7a8791d 2024-11-25T03:36:56,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/f46bc099f04745e3930cd831c7a8791d, entries=150, sequenceid=75, filesize=11.7 K 2024-11-25T03:36:56,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/7a1df036c47b402db6d29ea4340f178f as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/7a1df036c47b402db6d29ea4340f178f 2024-11-25T03:36:56,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:56,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505876738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:56,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:56,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:56,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:56,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:56,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505876738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:56,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:56,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:56,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:56,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505876738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:56,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:56,748 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/7a1df036c47b402db6d29ea4340f178f, entries=150, sequenceid=75, filesize=11.7 K 2024-11-25T03:36:56,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:56,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505876739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:56,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:56,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/59c9fa86e7394eef8e0768ba0efb37d7 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/59c9fa86e7394eef8e0768ba0efb37d7 2024-11-25T03:36:56,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:56,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:56,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:56,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:56,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:56,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/59c9fa86e7394eef8e0768ba0efb37d7, entries=150, sequenceid=75, filesize=11.7 K 2024-11-25T03:36:56,773 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for ee186277170c2bdfb070a7c209286857 in 733ms, sequenceid=75, compaction requested=false 2024-11-25T03:36:56,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:36:56,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-25T03:36:56,808 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:36:56,808 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:36:56,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-25T03:36:56,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-25T03:36:56,814 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:36:56,816 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:36:56,816 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:36:56,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-25T03:36:56,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-25T03:36:56,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:36:56,973 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing ee186277170c2bdfb070a7c209286857 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-25T03:36:56,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/8ff3acbe06de44d4910f0f40bac50e4b is 50, key is test_row_0/A:col10/1732505816076/Put/seqid=0 2024-11-25T03:36:57,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741852_1028 (size=12001) 2024-11-25T03:36:57,042 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/8ff3acbe06de44d4910f0f40bac50e4b 2024-11-25T03:36:57,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/61e2ee120a214a968d0a8e8da7cde3e8 is 50, key is test_row_0/B:col10/1732505816076/Put/seqid=0 2024-11-25T03:36:57,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-25T03:36:57,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741853_1029 (size=12001) 2024-11-25T03:36:57,141 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/61e2ee120a214a968d0a8e8da7cde3e8 2024-11-25T03:36:57,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/4deb5dcde47947b9b8479b16dc3497ff is 50, key is test_row_0/C:col10/1732505816076/Put/seqid=0 2024-11-25T03:36:57,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741854_1030 (size=12001) 2024-11-25T03:36:57,189 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/4deb5dcde47947b9b8479b16dc3497ff 2024-11-25T03:36:57,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/8ff3acbe06de44d4910f0f40bac50e4b as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/8ff3acbe06de44d4910f0f40bac50e4b 2024-11-25T03:36:57,212 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/8ff3acbe06de44d4910f0f40bac50e4b, entries=150, sequenceid=92, filesize=11.7 K 2024-11-25T03:36:57,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/61e2ee120a214a968d0a8e8da7cde3e8 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/61e2ee120a214a968d0a8e8da7cde3e8 2024-11-25T03:36:57,223 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/61e2ee120a214a968d0a8e8da7cde3e8, entries=150, sequenceid=92, filesize=11.7 K 2024-11-25T03:36:57,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/4deb5dcde47947b9b8479b16dc3497ff as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/4deb5dcde47947b9b8479b16dc3497ff 2024-11-25T03:36:57,235 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/4deb5dcde47947b9b8479b16dc3497ff, entries=150, sequenceid=92, filesize=11.7 K 2024-11-25T03:36:57,237 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=6.71 KB/6870 for ee186277170c2bdfb070a7c209286857 in 264ms, sequenceid=92, compaction requested=true 2024-11-25T03:36:57,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:36:57,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:36:57,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-25T03:36:57,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-25T03:36:57,243 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-25T03:36:57,244 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 424 msec 2024-11-25T03:36:57,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 437 msec 2024-11-25T03:36:57,283 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:36:57,283 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee186277170c2bdfb070a7c209286857 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-25T03:36:57,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on ee186277170c2bdfb070a7c209286857 2024-11-25T03:36:57,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/2e4dd8ef19cf44a7a4767df5d4225ddb is 50, key is test_row_0/A:col10/1732505817265/Put/seqid=0 2024-11-25T03:36:57,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741855_1031 (size=14341) 2024-11-25T03:36:57,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:57,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505877365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:57,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:57,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:57,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:57,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:57,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505877367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:57,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:57,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:57,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:57,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:57,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505877369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:57,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:57,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:57,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:57,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:57,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505877378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:57,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:57,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:57,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:57,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:57,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505877378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:57,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:57,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:57,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:57,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-25T03:36:57,448 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:36:57,449 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:36:57,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-25T03:36:57,456 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:36:57,457 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:36:57,458 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:36:57,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-25T03:36:57,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:57,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505877487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:57,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:57,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505877487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:57,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:57,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:57,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:57,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:57,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:57,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:57,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:57,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505877498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:57,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:57,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:57,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:57,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:57,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505877498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:57,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:57,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:57,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:57,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:57,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505877507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:57,521 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:57,521 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:57,521 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:57,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-25T03:36:57,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-25T03:36:57,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:36:57,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. as already flushing 2024-11-25T03:36:57,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:36:57,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:36:57,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:36:57,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:36:57,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:57,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505877708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:57,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:57,712 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/2e4dd8ef19cf44a7a4767df5d4225ddb 2024-11-25T03:36:57,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505877708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:57,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:57,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:57,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:57,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:57,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:57,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:57,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:57,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505877708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:57,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:57,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:57,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:57,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:57,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505877717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:57,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:57,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:57,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:57,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:57,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505877727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:57,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:57,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:57,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:57,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/ff94de45bc1846089a69d16458dfc752 is 50, key is test_row_0/B:col10/1732505817265/Put/seqid=0 2024-11-25T03:36:57,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741856_1032 (size=12001) 2024-11-25T03:36:57,761 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/ff94de45bc1846089a69d16458dfc752 2024-11-25T03:36:57,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-25T03:36:57,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:36:57,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. as already flushing 2024-11-25T03:36:57,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:36:57,775 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:36:57,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:36:57,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:36:57,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-25T03:36:57,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/cbcaaad6dfc84fdc8930a6216e49528d is 50, key is test_row_0/C:col10/1732505817265/Put/seqid=0 2024-11-25T03:36:57,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741857_1033 (size=12001) 2024-11-25T03:36:57,809 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/cbcaaad6dfc84fdc8930a6216e49528d 2024-11-25T03:36:57,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/2e4dd8ef19cf44a7a4767df5d4225ddb as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/2e4dd8ef19cf44a7a4767df5d4225ddb 2024-11-25T03:36:57,837 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/2e4dd8ef19cf44a7a4767df5d4225ddb, entries=200, sequenceid=103, filesize=14.0 K 2024-11-25T03:36:57,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/ff94de45bc1846089a69d16458dfc752 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/ff94de45bc1846089a69d16458dfc752 2024-11-25T03:36:57,850 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/ff94de45bc1846089a69d16458dfc752, entries=150, sequenceid=103, filesize=11.7 K 2024-11-25T03:36:57,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/cbcaaad6dfc84fdc8930a6216e49528d as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/cbcaaad6dfc84fdc8930a6216e49528d 2024-11-25T03:36:57,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/cbcaaad6dfc84fdc8930a6216e49528d, entries=150, sequenceid=103, filesize=11.7 K 2024-11-25T03:36:57,875 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for ee186277170c2bdfb070a7c209286857 in 592ms, sequenceid=103, compaction requested=true 2024-11-25T03:36:57,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:36:57,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee186277170c2bdfb070a7c209286857:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:36:57,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:36:57,876 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:36:57,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee186277170c2bdfb070a7c209286857:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:36:57,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:36:57,876 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:36:57,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee186277170c2bdfb070a7c209286857:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:36:57,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:36:57,879 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50447 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:36:57,879 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:36:57,880 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): ee186277170c2bdfb070a7c209286857/A is initiating minor compaction (all files) 2024-11-25T03:36:57,880 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): ee186277170c2bdfb070a7c209286857/B is initiating minor compaction (all files) 2024-11-25T03:36:57,880 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee186277170c2bdfb070a7c209286857/A in TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:36:57,880 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee186277170c2bdfb070a7c209286857/B in TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:36:57,880 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/4ccec1bd1585433aa572d844cca23ddb, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/f46bc099f04745e3930cd831c7a8791d, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/8ff3acbe06de44d4910f0f40bac50e4b, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/2e4dd8ef19cf44a7a4767df5d4225ddb] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp, totalSize=49.3 K 2024-11-25T03:36:57,880 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/7dfc7fe3067d44e4a3c122615f020c60, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/7a1df036c47b402db6d29ea4340f178f, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/61e2ee120a214a968d0a8e8da7cde3e8, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/ff94de45bc1846089a69d16458dfc752] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp, totalSize=47.0 K 2024-11-25T03:36:57,882 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 7dfc7fe3067d44e4a3c122615f020c60, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732505814141 2024-11-25T03:36:57,882 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ccec1bd1585433aa572d844cca23ddb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732505814141 2024-11-25T03:36:57,882 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a1df036c47b402db6d29ea4340f178f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732505815384 2024-11-25T03:36:57,883 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting f46bc099f04745e3930cd831c7a8791d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732505815384 2024-11-25T03:36:57,883 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 61e2ee120a214a968d0a8e8da7cde3e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732505816070 2024-11-25T03:36:57,884 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ff3acbe06de44d4910f0f40bac50e4b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732505816070 2024-11-25T03:36:57,884 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting ff94de45bc1846089a69d16458dfc752, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732505817265 2024-11-25T03:36:57,884 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e4dd8ef19cf44a7a4767df5d4225ddb, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732505817259 2024-11-25T03:36:57,913 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee186277170c2bdfb070a7c209286857#B#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:36:57,914 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/c9949a29bd76499c9751e8876d3fae83 is 50, key is test_row_0/B:col10/1732505817265/Put/seqid=0 2024-11-25T03:36:57,915 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee186277170c2bdfb070a7c209286857#A#compaction#22 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:36:57,916 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/f85b014f3c6d467fa0ac2c7b9fecca1f is 50, key is test_row_0/A:col10/1732505817265/Put/seqid=0 2024-11-25T03:36:57,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-25T03:36:57,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:36:57,931 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing ee186277170c2bdfb070a7c209286857 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-25T03:36:57,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741858_1034 (size=12241) 2024-11-25T03:36:57,948 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/c9949a29bd76499c9751e8876d3fae83 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/c9949a29bd76499c9751e8876d3fae83 2024-11-25T03:36:57,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/f2e1ce89e5b24173bd82727e71dd097e is 50, key is test_row_0/A:col10/1732505817364/Put/seqid=0 2024-11-25T03:36:57,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741859_1035 (size=12241) 2024-11-25T03:36:57,960 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee186277170c2bdfb070a7c209286857/B of ee186277170c2bdfb070a7c209286857 into c9949a29bd76499c9751e8876d3fae83(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:36:57,960 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:36:57,960 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., storeName=ee186277170c2bdfb070a7c209286857/B, priority=12, startTime=1732505817876; duration=0sec 2024-11-25T03:36:57,961 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:36:57,961 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee186277170c2bdfb070a7c209286857:B 2024-11-25T03:36:57,961 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:36:57,964 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:36:57,964 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): ee186277170c2bdfb070a7c209286857/C is initiating minor compaction (all files) 2024-11-25T03:36:57,964 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee186277170c2bdfb070a7c209286857/C in TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:36:57,964 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/ae61d993d0f044b79e81ee41db2933cb, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/59c9fa86e7394eef8e0768ba0efb37d7, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/4deb5dcde47947b9b8479b16dc3497ff, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/cbcaaad6dfc84fdc8930a6216e49528d] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp, totalSize=47.0 K 2024-11-25T03:36:57,965 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting ae61d993d0f044b79e81ee41db2933cb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732505814141 2024-11-25T03:36:57,966 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 59c9fa86e7394eef8e0768ba0efb37d7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732505815384 2024-11-25T03:36:57,966 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 4deb5dcde47947b9b8479b16dc3497ff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732505816070 2024-11-25T03:36:57,967 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting cbcaaad6dfc84fdc8930a6216e49528d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732505817265 2024-11-25T03:36:57,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741860_1036 (size=12001) 2024-11-25T03:36:57,984 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/f2e1ce89e5b24173bd82727e71dd097e 2024-11-25T03:36:57,995 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee186277170c2bdfb070a7c209286857#C#compaction#24 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:36:57,996 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/d2defad255684ece8fd2d6f5d9d07a17 is 50, key is test_row_0/C:col10/1732505817265/Put/seqid=0 2024-11-25T03:36:58,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/ab4376d67d74440e931def5856999ff0 is 50, key is test_row_0/B:col10/1732505817364/Put/seqid=0 2024-11-25T03:36:58,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on ee186277170c2bdfb070a7c209286857 2024-11-25T03:36:58,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. as already flushing 2024-11-25T03:36:58,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:58,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4773) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4699) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5194) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5188) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5184) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3333) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:58,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505878038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:58,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4773) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4699) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5194) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5188) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5184) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3333) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:58,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4773) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4699) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5194) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5188) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5184) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3333) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:58,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:58,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:58,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505878039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:58,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741861_1037 (size=12241) 2024-11-25T03:36:58,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:58,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:58,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:58,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741862_1038 (size=12001) 2024-11-25T03:36:58,057 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/ab4376d67d74440e931def5856999ff0 2024-11-25T03:36:58,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505878039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:58,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:58,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:58,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:58,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:58,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505878048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:58,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:58,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505878048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:58,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:58,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:58,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:58,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:58,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:58,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:58,072 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/d2defad255684ece8fd2d6f5d9d07a17 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/d2defad255684ece8fd2d6f5d9d07a17 2024-11-25T03:36:58,086 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee186277170c2bdfb070a7c209286857/C of ee186277170c2bdfb070a7c209286857 into d2defad255684ece8fd2d6f5d9d07a17(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:36:58,086 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:36:58,087 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., storeName=ee186277170c2bdfb070a7c209286857/C, priority=12, startTime=1732505817876; duration=0sec 2024-11-25T03:36:58,087 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:36:58,087 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee186277170c2bdfb070a7c209286857:C 2024-11-25T03:36:58,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-25T03:36:58,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/597414e26bc34972a20801a5906dddf0 is 50, key is test_row_0/C:col10/1732505817364/Put/seqid=0 2024-11-25T03:36:58,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741863_1039 (size=12001) 2024-11-25T03:36:58,119 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/597414e26bc34972a20801a5906dddf0 2024-11-25T03:36:58,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/f2e1ce89e5b24173bd82727e71dd097e as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/f2e1ce89e5b24173bd82727e71dd097e 2024-11-25T03:36:58,139 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/f2e1ce89e5b24173bd82727e71dd097e, entries=150, sequenceid=128, filesize=11.7 K 2024-11-25T03:36:58,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/ab4376d67d74440e931def5856999ff0 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/ab4376d67d74440e931def5856999ff0 2024-11-25T03:36:58,151 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/ab4376d67d74440e931def5856999ff0, entries=150, sequenceid=128, filesize=11.7 K 2024-11-25T03:36:58,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/597414e26bc34972a20801a5906dddf0 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/597414e26bc34972a20801a5906dddf0 2024-11-25T03:36:58,162 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/597414e26bc34972a20801a5906dddf0, entries=150, sequenceid=128, filesize=11.7 K 2024-11-25T03:36:58,164 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for ee186277170c2bdfb070a7c209286857 in 233ms, sequenceid=128, compaction requested=false 2024-11-25T03:36:58,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:36:58,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:36:58,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-25T03:36:58,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-25T03:36:58,175 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-25T03:36:58,175 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 712 msec 2024-11-25T03:36:58,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on ee186277170c2bdfb070a7c209286857 2024-11-25T03:36:58,179 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:36:58,179 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee186277170c2bdfb070a7c209286857 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-25T03:36:58,184 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 731 msec 2024-11-25T03:36:58,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/8962bbe8a13246aa81b5bdccc983867c is 50, key is test_row_0/A:col10/1732505818171/Put/seqid=0 2024-11-25T03:36:58,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741864_1040 (size=12151) 2024-11-25T03:36:58,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:58,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505878246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:58,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:58,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:58,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:58,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:58,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505878252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:58,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:58,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:58,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:58,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:58,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505878252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:58,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:58,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:58,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:58,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:58,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505878359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:58,362 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:58,362 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:58,362 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:58,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:58,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505878368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:58,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:58,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505878368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:58,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:58,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:58,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:58,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:58,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:58,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:58,381 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/f85b014f3c6d467fa0ac2c7b9fecca1f as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/f85b014f3c6d467fa0ac2c7b9fecca1f 2024-11-25T03:36:58,397 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee186277170c2bdfb070a7c209286857/A of ee186277170c2bdfb070a7c209286857 into f85b014f3c6d467fa0ac2c7b9fecca1f(size=12.0 K), total size for store is 23.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:36:58,397 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:36:58,397 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., storeName=ee186277170c2bdfb070a7c209286857/A, priority=12, startTime=1732505817876; duration=0sec 2024-11-25T03:36:58,398 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:36:58,398 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee186277170c2bdfb070a7c209286857:A 2024-11-25T03:36:58,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:58,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505878558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:58,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:58,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:58,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:58,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:58,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505878558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:58,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:58,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:58,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:58,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:58,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505878567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:58,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:58,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:58,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:58,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:58,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505878588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:58,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:58,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:58,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:58,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:58,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505878592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:58,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:58,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:58,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:58,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-25T03:36:58,597 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:36:58,598 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:36:58,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-25T03:36:58,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-25T03:36:58,602 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:36:58,604 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:36:58,604 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:36:58,645 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/8962bbe8a13246aa81b5bdccc983867c 2024-11-25T03:36:58,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/33c399cae6a9488e891c133857143039 is 50, key is test_row_0/B:col10/1732505818171/Put/seqid=0 2024-11-25T03:36:58,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741865_1041 (size=12151) 2024-11-25T03:36:58,685 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/33c399cae6a9488e891c133857143039 2024-11-25T03:36:58,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-25T03:36:58,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/dba3bc2703354402999a5d7da99de637 is 50, key is test_row_0/C:col10/1732505818171/Put/seqid=0 2024-11-25T03:36:58,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741866_1042 (size=12151) 2024-11-25T03:36:58,743 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/dba3bc2703354402999a5d7da99de637 2024-11-25T03:36:58,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/8962bbe8a13246aa81b5bdccc983867c as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/8962bbe8a13246aa81b5bdccc983867c 2024-11-25T03:36:58,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-25T03:36:58,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:36:58,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. as already flushing 2024-11-25T03:36:58,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:36:58,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:36:58,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:36:58,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:36:58,776 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/8962bbe8a13246aa81b5bdccc983867c, entries=150, sequenceid=144, filesize=11.9 K 2024-11-25T03:36:58,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/33c399cae6a9488e891c133857143039 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/33c399cae6a9488e891c133857143039 2024-11-25T03:36:58,790 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/33c399cae6a9488e891c133857143039, entries=150, sequenceid=144, filesize=11.9 K 2024-11-25T03:36:58,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/dba3bc2703354402999a5d7da99de637 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/dba3bc2703354402999a5d7da99de637 2024-11-25T03:36:58,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/dba3bc2703354402999a5d7da99de637, entries=150, sequenceid=144, filesize=11.9 K 2024-11-25T03:36:58,809 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for ee186277170c2bdfb070a7c209286857 in 630ms, sequenceid=144, compaction requested=true 2024-11-25T03:36:58,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:36:58,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee186277170c2bdfb070a7c209286857:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:36:58,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:36:58,810 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:36:58,810 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:36:58,813 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:36:58,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee186277170c2bdfb070a7c209286857:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:36:58,813 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): ee186277170c2bdfb070a7c209286857/A is initiating minor compaction (all files) 2024-11-25T03:36:58,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:36:58,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee186277170c2bdfb070a7c209286857:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:36:58,814 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee186277170c2bdfb070a7c209286857/A in TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:36:58,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:36:58,814 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/f85b014f3c6d467fa0ac2c7b9fecca1f, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/f2e1ce89e5b24173bd82727e71dd097e, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/8962bbe8a13246aa81b5bdccc983867c] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp, totalSize=35.5 K 2024-11-25T03:36:58,815 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:36:58,815 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): ee186277170c2bdfb070a7c209286857/B is initiating minor compaction (all files) 2024-11-25T03:36:58,815 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee186277170c2bdfb070a7c209286857/B in TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:36:58,816 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/c9949a29bd76499c9751e8876d3fae83, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/ab4376d67d74440e931def5856999ff0, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/33c399cae6a9488e891c133857143039] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp, totalSize=35.5 K 2024-11-25T03:36:58,816 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting f85b014f3c6d467fa0ac2c7b9fecca1f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732505817265 2024-11-25T03:36:58,816 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting c9949a29bd76499c9751e8876d3fae83, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732505817265 2024-11-25T03:36:58,817 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2e1ce89e5b24173bd82727e71dd097e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732505817359 2024-11-25T03:36:58,817 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting ab4376d67d74440e931def5856999ff0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732505817359 2024-11-25T03:36:58,818 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8962bbe8a13246aa81b5bdccc983867c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732505818042 2024-11-25T03:36:58,819 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 33c399cae6a9488e891c133857143039, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732505818042 2024-11-25T03:36:58,847 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee186277170c2bdfb070a7c209286857#A#compaction#30 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:36:58,848 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/d75be72d1b33439a8bccce9798519d89 is 50, key is test_row_0/A:col10/1732505818171/Put/seqid=0 2024-11-25T03:36:58,858 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee186277170c2bdfb070a7c209286857#B#compaction#31 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:36:58,859 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/4b8dbae4870c4fc0890209df19998adf is 50, key is test_row_0/B:col10/1732505818171/Put/seqid=0 2024-11-25T03:36:58,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on ee186277170c2bdfb070a7c209286857 2024-11-25T03:36:58,882 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:36:58,882 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee186277170c2bdfb070a7c209286857 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-25T03:36:58,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741867_1043 (size=12493) 2024-11-25T03:36:58,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741868_1044 (size=12493) 2024-11-25T03:36:58,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-25T03:36:58,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:36:58,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. as already flushing 2024-11-25T03:36:58,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:36:58,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:36:58,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:36:58,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:36:58,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/b5202a1881134fcf83d17717ffeae3db is 50, key is test_row_0/A:col10/1732505818230/Put/seqid=0 2024-11-25T03:36:58,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-25T03:36:58,920 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/d75be72d1b33439a8bccce9798519d89 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/d75be72d1b33439a8bccce9798519d89 2024-11-25T03:36:58,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:58,932 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/4b8dbae4870c4fc0890209df19998adf as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/4b8dbae4870c4fc0890209df19998adf 2024-11-25T03:36:58,933 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee186277170c2bdfb070a7c209286857/A of ee186277170c2bdfb070a7c209286857 into d75be72d1b33439a8bccce9798519d89(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:36:58,933 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:36:58,933 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., storeName=ee186277170c2bdfb070a7c209286857/A, priority=13, startTime=1732505818809; duration=0sec 2024-11-25T03:36:58,933 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:36:58,933 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee186277170c2bdfb070a7c209286857:A 2024-11-25T03:36:58,933 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:36:58,935 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:36:58,936 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): ee186277170c2bdfb070a7c209286857/C is initiating minor compaction (all files) 2024-11-25T03:36:58,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505878922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:58,936 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee186277170c2bdfb070a7c209286857/C in TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:36:58,937 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/d2defad255684ece8fd2d6f5d9d07a17, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/597414e26bc34972a20801a5906dddf0, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/dba3bc2703354402999a5d7da99de637] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp, totalSize=35.5 K 2024-11-25T03:36:58,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:58,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:58,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:58,939 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2defad255684ece8fd2d6f5d9d07a17, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732505817265 2024-11-25T03:36:58,942 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 597414e26bc34972a20801a5906dddf0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732505817359 2024-11-25T03:36:58,942 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee186277170c2bdfb070a7c209286857/B of ee186277170c2bdfb070a7c209286857 into 4b8dbae4870c4fc0890209df19998adf(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:36:58,942 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:36:58,942 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., storeName=ee186277170c2bdfb070a7c209286857/B, priority=13, startTime=1732505818810; duration=0sec 2024-11-25T03:36:58,943 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:36:58,943 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee186277170c2bdfb070a7c209286857:B 2024-11-25T03:36:58,943 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting dba3bc2703354402999a5d7da99de637, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732505818042 2024-11-25T03:36:58,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:58,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505878932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:58,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:58,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:58,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:58,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:58,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505878932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:58,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:58,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:58,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:58,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741869_1045 (size=14541) 2024-11-25T03:36:58,963 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee186277170c2bdfb070a7c209286857#C#compaction#33 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:36:58,964 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/196dd33618d446adbd71ad3c519dc395 is 50, key is test_row_0/C:col10/1732505818171/Put/seqid=0 2024-11-25T03:36:58,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741870_1046 (size=12493) 2024-11-25T03:36:59,012 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/196dd33618d446adbd71ad3c519dc395 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/196dd33618d446adbd71ad3c519dc395 2024-11-25T03:36:59,041 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee186277170c2bdfb070a7c209286857/C of ee186277170c2bdfb070a7c209286857 into 196dd33618d446adbd71ad3c519dc395(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:36:59,041 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:36:59,041 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., storeName=ee186277170c2bdfb070a7c209286857/C, priority=13, startTime=1732505818813; duration=0sec 2024-11-25T03:36:59,041 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:36:59,041 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee186277170c2bdfb070a7c209286857:C 2024-11-25T03:36:59,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:59,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505879048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:59,051 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:59,051 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:59,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:59,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:59,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505879058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:59,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:59,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:59,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:59,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:59,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505879058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:59,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:59,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:59,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:59,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-25T03:36:59,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:36:59,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. as already flushing 2024-11-25T03:36:59,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:36:59,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:36:59,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:36:59,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:36:59,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-25T03:36:59,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:36:59,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. as already flushing 2024-11-25T03:36:59,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:36:59,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:36:59,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:36:59,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:36:59,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-25T03:36:59,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:59,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505879257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:59,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:59,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:59,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:59,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:59,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505879268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:59,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:59,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505879268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:59,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:59,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:59,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:59,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:59,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:59,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:59,349 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/b5202a1881134fcf83d17717ffeae3db 2024-11-25T03:36:59,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/ea80579fde3f4ee0a5d7389807668626 is 50, key is test_row_0/B:col10/1732505818230/Put/seqid=0 2024-11-25T03:36:59,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-25T03:36:59,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:36:59,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. as already flushing 2024-11-25T03:36:59,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:36:59,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:36:59,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:36:59,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:36:59,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741871_1047 (size=12151) 2024-11-25T03:36:59,420 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/ea80579fde3f4ee0a5d7389807668626 2024-11-25T03:36:59,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/629076da23744d77989150735e5ddb3d is 50, key is test_row_0/C:col10/1732505818230/Put/seqid=0 2024-11-25T03:36:59,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741872_1048 (size=12151) 2024-11-25T03:36:59,470 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/629076da23744d77989150735e5ddb3d 2024-11-25T03:36:59,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/b5202a1881134fcf83d17717ffeae3db as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/b5202a1881134fcf83d17717ffeae3db 2024-11-25T03:36:59,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/b5202a1881134fcf83d17717ffeae3db, entries=200, sequenceid=169, filesize=14.2 K 2024-11-25T03:36:59,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/ea80579fde3f4ee0a5d7389807668626 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/ea80579fde3f4ee0a5d7389807668626 2024-11-25T03:36:59,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/ea80579fde3f4ee0a5d7389807668626, entries=150, sequenceid=169, filesize=11.9 K 2024-11-25T03:36:59,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/629076da23744d77989150735e5ddb3d as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/629076da23744d77989150735e5ddb3d 2024-11-25T03:36:59,517 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/629076da23744d77989150735e5ddb3d, entries=150, sequenceid=169, filesize=11.9 K 2024-11-25T03:36:59,520 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for ee186277170c2bdfb070a7c209286857 in 638ms, sequenceid=169, compaction requested=false 2024-11-25T03:36:59,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:36:59,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-25T03:36:59,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:36:59,536 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing ee186277170c2bdfb070a7c209286857 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-25T03:36:59,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/8738b017b0bc467484c9a06327421192 is 50, key is test_row_0/A:col10/1732505818907/Put/seqid=0 2024-11-25T03:36:59,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on ee186277170c2bdfb070a7c209286857 2024-11-25T03:36:59,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. as already flushing 2024-11-25T03:36:59,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741873_1049 (size=12151) 2024-11-25T03:36:59,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:59,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505879613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:59,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:59,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505879614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:59,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:59,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:59,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:59,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:59,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:59,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:59,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:59,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505879616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:59,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:59,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505879619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:59,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:59,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505879627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:59,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:59,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:59,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:59,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:59,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:59,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:59,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:59,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:59,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:59,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-25T03:36:59,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:59,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505879737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:59,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:59,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:59,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:59,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:59,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505879738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:59,743 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:59,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:59,743 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:59,743 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:59,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505879738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:59,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:59,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:59,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:59,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:59,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505879738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:59,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:59,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505879738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:59,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:59,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:59,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:59,745 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:59,745 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:59,745 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:59,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:59,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505879948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:59,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:59,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:59,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:59,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:59,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505879953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:59,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:59,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:59,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:59,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:59,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505879953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:59,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:59,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:59,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:59,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:59,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505879953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:59,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:59,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:59,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:59,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:36:59,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505879958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:36:59,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:59,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:36:59,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:36:59,989 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/8738b017b0bc467484c9a06327421192 2024-11-25T03:37:00,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/98c7909d282f410da21ad762115f771b is 50, key is test_row_0/B:col10/1732505818907/Put/seqid=0 2024-11-25T03:37:00,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741874_1050 (size=12151) 2024-11-25T03:37:00,023 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/98c7909d282f410da21ad762115f771b 2024-11-25T03:37:00,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/30543d4e604d4986bd606b6888865203 is 50, key is test_row_0/C:col10/1732505818907/Put/seqid=0 2024-11-25T03:37:00,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741875_1051 (size=12151) 2024-11-25T03:37:00,078 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/30543d4e604d4986bd606b6888865203 2024-11-25T03:37:00,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/8738b017b0bc467484c9a06327421192 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/8738b017b0bc467484c9a06327421192 2024-11-25T03:37:00,099 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/8738b017b0bc467484c9a06327421192, entries=150, sequenceid=184, filesize=11.9 K 2024-11-25T03:37:00,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/98c7909d282f410da21ad762115f771b as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/98c7909d282f410da21ad762115f771b 2024-11-25T03:37:00,113 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/98c7909d282f410da21ad762115f771b, entries=150, sequenceid=184, filesize=11.9 K 2024-11-25T03:37:00,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/30543d4e604d4986bd606b6888865203 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/30543d4e604d4986bd606b6888865203 2024-11-25T03:37:00,126 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/30543d4e604d4986bd606b6888865203, entries=150, sequenceid=184, filesize=11.9 K 2024-11-25T03:37:00,130 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for ee186277170c2bdfb070a7c209286857 in 592ms, sequenceid=184, compaction requested=true 2024-11-25T03:37:00,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:37:00,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:00,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-25T03:37:00,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-25T03:37:00,136 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-25T03:37:00,136 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5290 sec 2024-11-25T03:37:00,143 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.5410 sec 2024-11-25T03:37:00,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on ee186277170c2bdfb070a7c209286857 2024-11-25T03:37:00,261 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:37:00,261 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee186277170c2bdfb070a7c209286857 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-25T03:37:00,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/90ddb124ef7a48f683ba7cde65a047a2 is 50, key is test_row_0/A:col10/1732505819614/Put/seqid=0 2024-11-25T03:37:00,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:00,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505880284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:00,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:00,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:00,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:00,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:00,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505880284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:00,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:00,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:00,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:00,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:00,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505880284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:00,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:00,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:00,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:00,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:00,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505880285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:00,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:00,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:00,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:00,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:00,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505880288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:00,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:00,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:00,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:00,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741876_1052 (size=14541) 2024-11-25T03:37:00,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/90ddb124ef7a48f683ba7cde65a047a2 2024-11-25T03:37:00,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/2807a04f71d44f8abac2d2e5aa09e188 is 50, key is test_row_0/B:col10/1732505819614/Put/seqid=0 2024-11-25T03:37:00,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741877_1053 (size=12151) 2024-11-25T03:37:00,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/2807a04f71d44f8abac2d2e5aa09e188 2024-11-25T03:37:00,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/3ed4b6443af54090a00862dd9df90b4b is 50, key is test_row_0/C:col10/1732505819614/Put/seqid=0 2024-11-25T03:37:00,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741878_1054 (size=12151) 2024-11-25T03:37:00,394 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/3ed4b6443af54090a00862dd9df90b4b 2024-11-25T03:37:00,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:00,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505880399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:00,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:00,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:00,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:00,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:00,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505880399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:00,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:00,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:00,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505880399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:00,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:00,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:00,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:00,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505880400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:00,403 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:00,403 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:00,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:00,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:00,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:00,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:00,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/90ddb124ef7a48f683ba7cde65a047a2 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/90ddb124ef7a48f683ba7cde65a047a2 2024-11-25T03:37:00,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:00,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505880410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:00,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:00,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:00,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:00,419 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/90ddb124ef7a48f683ba7cde65a047a2, entries=200, sequenceid=209, filesize=14.2 K 2024-11-25T03:37:00,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/2807a04f71d44f8abac2d2e5aa09e188 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/2807a04f71d44f8abac2d2e5aa09e188 2024-11-25T03:37:00,428 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/2807a04f71d44f8abac2d2e5aa09e188, entries=150, sequenceid=209, filesize=11.9 K 2024-11-25T03:37:00,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/3ed4b6443af54090a00862dd9df90b4b as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/3ed4b6443af54090a00862dd9df90b4b 2024-11-25T03:37:00,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/3ed4b6443af54090a00862dd9df90b4b, entries=150, sequenceid=209, filesize=11.9 K 2024-11-25T03:37:00,439 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for ee186277170c2bdfb070a7c209286857 in 178ms, sequenceid=209, compaction requested=true 2024-11-25T03:37:00,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:37:00,442 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:37:00,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee186277170c2bdfb070a7c209286857:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:37:00,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:00,445 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:37:00,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee186277170c2bdfb070a7c209286857:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:37:00,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:00,446 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53726 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:37:00,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee186277170c2bdfb070a7c209286857:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:37:00,446 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): ee186277170c2bdfb070a7c209286857/A is initiating minor compaction (all files) 2024-11-25T03:37:00,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:37:00,446 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee186277170c2bdfb070a7c209286857/A in TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:00,446 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/d75be72d1b33439a8bccce9798519d89, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/b5202a1881134fcf83d17717ffeae3db, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/8738b017b0bc467484c9a06327421192, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/90ddb124ef7a48f683ba7cde65a047a2] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp, totalSize=52.5 K 2024-11-25T03:37:00,447 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting d75be72d1b33439a8bccce9798519d89, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732505818042 2024-11-25T03:37:00,447 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:37:00,448 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): ee186277170c2bdfb070a7c209286857/B is initiating minor compaction (all files) 2024-11-25T03:37:00,448 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee186277170c2bdfb070a7c209286857/B in TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:00,448 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting b5202a1881134fcf83d17717ffeae3db, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732505818230 2024-11-25T03:37:00,448 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/4b8dbae4870c4fc0890209df19998adf, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/ea80579fde3f4ee0a5d7389807668626, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/98c7909d282f410da21ad762115f771b, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/2807a04f71d44f8abac2d2e5aa09e188] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp, totalSize=47.8 K 2024-11-25T03:37:00,448 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 8738b017b0bc467484c9a06327421192, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1732505818907 2024-11-25T03:37:00,448 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b8dbae4870c4fc0890209df19998adf, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732505818042 2024-11-25T03:37:00,449 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 90ddb124ef7a48f683ba7cde65a047a2, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732505819610 2024-11-25T03:37:00,449 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea80579fde3f4ee0a5d7389807668626, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732505818230 2024-11-25T03:37:00,450 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98c7909d282f410da21ad762115f771b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1732505818907 2024-11-25T03:37:00,450 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2807a04f71d44f8abac2d2e5aa09e188, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732505819614 2024-11-25T03:37:00,479 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee186277170c2bdfb070a7c209286857#A#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:37:00,480 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/343b88cc95db4a36b10c3ae5f0b80fc0 is 50, key is test_row_0/A:col10/1732505819614/Put/seqid=0 2024-11-25T03:37:00,493 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee186277170c2bdfb070a7c209286857#B#compaction#43 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:37:00,494 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/8d4bbebed0324bcd9ddd2e014efec18b is 50, key is test_row_0/B:col10/1732505819614/Put/seqid=0 2024-11-25T03:37:00,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741880_1056 (size=12629) 2024-11-25T03:37:00,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741879_1055 (size=12629) 2024-11-25T03:37:00,579 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/8d4bbebed0324bcd9ddd2e014efec18b as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/8d4bbebed0324bcd9ddd2e014efec18b 2024-11-25T03:37:00,582 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/343b88cc95db4a36b10c3ae5f0b80fc0 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/343b88cc95db4a36b10c3ae5f0b80fc0 2024-11-25T03:37:00,595 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee186277170c2bdfb070a7c209286857/B of ee186277170c2bdfb070a7c209286857 into 8d4bbebed0324bcd9ddd2e014efec18b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:37:00,595 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee186277170c2bdfb070a7c209286857/A of ee186277170c2bdfb070a7c209286857 into 343b88cc95db4a36b10c3ae5f0b80fc0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:37:00,595 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:37:00,595 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:37:00,595 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., storeName=ee186277170c2bdfb070a7c209286857/A, priority=12, startTime=1732505820442; duration=0sec 2024-11-25T03:37:00,595 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., storeName=ee186277170c2bdfb070a7c209286857/B, priority=12, startTime=1732505820445; duration=0sec 2024-11-25T03:37:00,595 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:37:00,595 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee186277170c2bdfb070a7c209286857:A 2024-11-25T03:37:00,595 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:00,595 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee186277170c2bdfb070a7c209286857:B 2024-11-25T03:37:00,595 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:37:00,597 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:37:00,597 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): ee186277170c2bdfb070a7c209286857/C is initiating minor compaction (all files) 2024-11-25T03:37:00,597 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee186277170c2bdfb070a7c209286857/C in TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:00,598 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/196dd33618d446adbd71ad3c519dc395, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/629076da23744d77989150735e5ddb3d, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/30543d4e604d4986bd606b6888865203, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/3ed4b6443af54090a00862dd9df90b4b] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp, totalSize=47.8 K 2024-11-25T03:37:00,599 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 196dd33618d446adbd71ad3c519dc395, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732505818042 2024-11-25T03:37:00,600 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 629076da23744d77989150735e5ddb3d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732505818230 2024-11-25T03:37:00,601 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 30543d4e604d4986bd606b6888865203, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1732505818907 2024-11-25T03:37:00,601 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ed4b6443af54090a00862dd9df90b4b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732505819614 2024-11-25T03:37:00,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on ee186277170c2bdfb070a7c209286857 2024-11-25T03:37:00,617 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:37:00,617 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee186277170c2bdfb070a7c209286857 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-25T03:37:00,622 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee186277170c2bdfb070a7c209286857#C#compaction#44 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:37:00,622 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/b1cdc4c3a05f4146844fc959665706b8 is 50, key is test_row_0/C:col10/1732505819614/Put/seqid=0 2024-11-25T03:37:00,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/a6f338444f0f459ca3403c7a501cc9a2 is 50, key is test_row_0/A:col10/1732505820614/Put/seqid=0 2024-11-25T03:37:00,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:00,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505880659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:00,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:00,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505880662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:00,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:00,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505880666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:00,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:00,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:00,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:00,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:00,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:00,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:00,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:00,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:00,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:00,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:00,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505880675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:00,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:00,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:00,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:00,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741881_1057 (size=12629) 2024-11-25T03:37:00,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741882_1058 (size=12151) 2024-11-25T03:37:00,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/a6f338444f0f459ca3403c7a501cc9a2 2024-11-25T03:37:00,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:00,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505880681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:00,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:00,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:00,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:00,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/5386e50525b24a2d8840f20fad1aa80a is 50, key is test_row_0/B:col10/1732505820614/Put/seqid=0 2024-11-25T03:37:00,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741883_1059 (size=12151) 2024-11-25T03:37:00,722 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/5386e50525b24a2d8840f20fad1aa80a 2024-11-25T03:37:00,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/9b56b55cb36544ddbd3630f1849197e1 is 50, key is test_row_0/C:col10/1732505820614/Put/seqid=0 2024-11-25T03:37:00,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-25T03:37:00,748 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:37:00,748 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:37:00,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-25T03:37:00,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-25T03:37:00,751 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:37:00,754 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:37:00,754 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:37:00,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741884_1060 (size=12151) 2024-11-25T03:37:00,775 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/9b56b55cb36544ddbd3630f1849197e1 2024-11-25T03:37:00,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/a6f338444f0f459ca3403c7a501cc9a2 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/a6f338444f0f459ca3403c7a501cc9a2 2024-11-25T03:37:00,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:00,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505880788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:00,793 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/a6f338444f0f459ca3403c7a501cc9a2, entries=150, sequenceid=225, filesize=11.9 K 2024-11-25T03:37:00,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:00,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:00,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:00,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:00,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:00,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505880789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:00,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505880789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:00,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:00,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:00,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:00,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:00,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:00,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:00,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:00,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505880789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:00,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:00,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:00,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:00,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/5386e50525b24a2d8840f20fad1aa80a as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/5386e50525b24a2d8840f20fad1aa80a 2024-11-25T03:37:00,803 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/5386e50525b24a2d8840f20fad1aa80a, entries=150, sequenceid=225, filesize=11.9 K 2024-11-25T03:37:00,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/9b56b55cb36544ddbd3630f1849197e1 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/9b56b55cb36544ddbd3630f1849197e1 2024-11-25T03:37:00,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:00,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505880807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:00,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:00,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:00,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:00,855 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/9b56b55cb36544ddbd3630f1849197e1, entries=150, sequenceid=225, filesize=11.9 K 2024-11-25T03:37:00,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-25T03:37:00,858 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for ee186277170c2bdfb070a7c209286857 in 240ms, sequenceid=225, compaction requested=false 2024-11-25T03:37:00,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:37:00,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-25T03:37:00,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:00,910 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing ee186277170c2bdfb070a7c209286857 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-25T03:37:00,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/88130f4e637e4847ac02fbab4290b511 is 50, key is test_row_0/A:col10/1732505820672/Put/seqid=0 2024-11-25T03:37:00,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741885_1061 (size=12151) 2024-11-25T03:37:00,949 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/88130f4e637e4847ac02fbab4290b511 2024-11-25T03:37:00,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/bded33507fa549c2bc29fc181238fb8f is 50, key is test_row_0/B:col10/1732505820672/Put/seqid=0 2024-11-25T03:37:01,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on ee186277170c2bdfb070a7c209286857 2024-11-25T03:37:01,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. as already flushing 2024-11-25T03:37:01,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741886_1062 (size=12151) 2024-11-25T03:37:01,022 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/bded33507fa549c2bc29fc181238fb8f 2024-11-25T03:37:01,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:01,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505881023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:01,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:01,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:01,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:01,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505881025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:01,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:01,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:01,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505881028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:01,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:01,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:01,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:01,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:01,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:01,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:01,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:01,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505881030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:01,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:01,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:01,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:01,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:01,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505881031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:01,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:01,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:01,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:01,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/b8c0eebc0a774bdd9fc502ab35e9f9d2 is 50, key is test_row_0/C:col10/1732505820672/Put/seqid=0 2024-11-25T03:37:01,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-25T03:37:01,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741887_1063 (size=12151) 2024-11-25T03:37:01,096 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/b1cdc4c3a05f4146844fc959665706b8 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/b1cdc4c3a05f4146844fc959665706b8 2024-11-25T03:37:01,106 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee186277170c2bdfb070a7c209286857/C of ee186277170c2bdfb070a7c209286857 into b1cdc4c3a05f4146844fc959665706b8(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:37:01,106 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:37:01,106 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., storeName=ee186277170c2bdfb070a7c209286857/C, priority=12, startTime=1732505820446; duration=0sec 2024-11-25T03:37:01,106 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:01,106 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee186277170c2bdfb070a7c209286857:C 2024-11-25T03:37:01,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:01,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505881138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:01,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:01,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:01,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:01,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:01,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505881138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:01,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:01,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:01,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:01,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:01,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505881138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:01,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:01,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:01,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:01,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:01,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505881138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:01,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:01,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:01,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:01,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:01,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505881148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:01,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:01,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:01,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:01,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:01,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505881347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:01,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:01,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:01,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:01,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:01,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505881348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:01,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:01,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:01,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:01,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:01,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505881357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:01,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:01,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:01,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:01,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:01,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:01,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505881357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:01,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505881357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:01,359 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:01,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:01,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:01,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:01,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:01,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:01,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-25T03:37:01,491 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/b8c0eebc0a774bdd9fc502ab35e9f9d2 2024-11-25T03:37:01,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/88130f4e637e4847ac02fbab4290b511 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/88130f4e637e4847ac02fbab4290b511 2024-11-25T03:37:01,513 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/88130f4e637e4847ac02fbab4290b511, entries=150, sequenceid=247, filesize=11.9 K 2024-11-25T03:37:01,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/bded33507fa549c2bc29fc181238fb8f as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/bded33507fa549c2bc29fc181238fb8f 2024-11-25T03:37:01,524 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/bded33507fa549c2bc29fc181238fb8f, entries=150, sequenceid=247, filesize=11.9 K 2024-11-25T03:37:01,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/b8c0eebc0a774bdd9fc502ab35e9f9d2 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/b8c0eebc0a774bdd9fc502ab35e9f9d2 2024-11-25T03:37:01,534 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/b8c0eebc0a774bdd9fc502ab35e9f9d2, entries=150, sequenceid=247, filesize=11.9 K 2024-11-25T03:37:01,535 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for ee186277170c2bdfb070a7c209286857 in 625ms, sequenceid=247, compaction requested=true 2024-11-25T03:37:01,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:37:01,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:01,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-25T03:37:01,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-25T03:37:01,546 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-25T03:37:01,547 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 788 msec 2024-11-25T03:37:01,550 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 800 msec 2024-11-25T03:37:01,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on ee186277170c2bdfb070a7c209286857 2024-11-25T03:37:01,664 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:37:01,664 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee186277170c2bdfb070a7c209286857 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-25T03:37:01,681 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/bedbe9c9b12341258acab2238fbfbb90 is 50, key is test_row_0/A:col10/1732505821014/Put/seqid=0 2024-11-25T03:37:01,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:01,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505881693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:01,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:01,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:01,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:01,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:01,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505881695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:01,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:01,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:01,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:01,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:01,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505881696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:01,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:01,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:01,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:01,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:01,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505881697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:01,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:01,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:01,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:01,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:01,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505881700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:01,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:01,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:01,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:01,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741888_1064 (size=14691) 2024-11-25T03:37:01,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:01,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505881808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:01,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:01,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:01,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:01,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:01,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:01,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505881808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:01,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505881808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:01,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:01,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:01,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:01,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:01,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505881809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:01,814 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:01,814 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:01,814 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:01,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:01,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:01,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:01,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:01,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505881818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:01,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:01,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:01,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:01,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-25T03:37:01,888 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:37:01,888 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:37:01,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-25T03:37:01,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-25T03:37:01,892 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:37:01,894 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:37:01,894 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:37:01,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-25T03:37:02,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:02,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:02,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505882018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:02,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505882018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:02,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:02,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:02,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:02,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:02,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:02,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:02,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:02,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505882018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:02,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:02,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:02,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:02,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:02,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505882018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:02,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:02,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:02,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:02,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:02,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505882027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:02,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:02,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:02,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:02,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-25T03:37:02,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:02,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. as already flushing 2024-11-25T03:37:02,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:02,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:02,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:02,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:02,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/bedbe9c9b12341258acab2238fbfbb90 2024-11-25T03:37:02,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/bb28620f0b45402c814d00b4dfd8e8e9 is 50, key is test_row_0/B:col10/1732505821014/Put/seqid=0 2024-11-25T03:37:02,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741889_1065 (size=12251) 2024-11-25T03:37:02,144 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/bb28620f0b45402c814d00b4dfd8e8e9 2024-11-25T03:37:02,157 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/25ead75a50df4ca3bad2d9419428d210 is 50, key is test_row_0/C:col10/1732505821014/Put/seqid=0 2024-11-25T03:37:02,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741890_1066 (size=12251) 2024-11-25T03:37:02,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-25T03:37:02,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:02,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. as already flushing 2024-11-25T03:37:02,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:02,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:02,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:02,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:02,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-25T03:37:02,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:02,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505882327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:02,330 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:02,330 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:02,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:02,330 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:02,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505882327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:02,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:02,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:02,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:02,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:02,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505882328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:02,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:02,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:02,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:02,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:02,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505882328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:02,331 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:02,331 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:02,331 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:02,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:02,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505882338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:02,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:02,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:02,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:02,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-25T03:37:02,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:02,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. as already flushing 2024-11-25T03:37:02,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:02,359 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:02,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:02,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:02,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-25T03:37:02,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:02,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. as already flushing 2024-11-25T03:37:02,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:02,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:02,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:02,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:02,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-25T03:37:02,571 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/25ead75a50df4ca3bad2d9419428d210 2024-11-25T03:37:02,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/bedbe9c9b12341258acab2238fbfbb90 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/bedbe9c9b12341258acab2238fbfbb90 2024-11-25T03:37:02,584 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/bedbe9c9b12341258acab2238fbfbb90, entries=200, sequenceid=265, filesize=14.3 K 2024-11-25T03:37:02,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/bb28620f0b45402c814d00b4dfd8e8e9 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/bb28620f0b45402c814d00b4dfd8e8e9 2024-11-25T03:37:02,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/bb28620f0b45402c814d00b4dfd8e8e9, entries=150, sequenceid=265, filesize=12.0 K 2024-11-25T03:37:02,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/25ead75a50df4ca3bad2d9419428d210 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/25ead75a50df4ca3bad2d9419428d210 2024-11-25T03:37:02,601 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/25ead75a50df4ca3bad2d9419428d210, entries=150, sequenceid=265, filesize=12.0 K 2024-11-25T03:37:02,602 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for ee186277170c2bdfb070a7c209286857 in 938ms, sequenceid=265, compaction requested=true 2024-11-25T03:37:02,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:37:02,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee186277170c2bdfb070a7c209286857:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:37:02,603 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:37:02,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:02,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee186277170c2bdfb070a7c209286857:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:37:02,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:02,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee186277170c2bdfb070a7c209286857:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:37:02,603 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:37:02,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:37:02,606 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51622 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:37:02,606 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49182 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:37:02,606 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): ee186277170c2bdfb070a7c209286857/A is initiating minor compaction (all files) 2024-11-25T03:37:02,606 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): ee186277170c2bdfb070a7c209286857/B is initiating minor compaction (all files) 2024-11-25T03:37:02,606 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee186277170c2bdfb070a7c209286857/A in TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:02,606 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee186277170c2bdfb070a7c209286857/B in TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:02,606 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/343b88cc95db4a36b10c3ae5f0b80fc0, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/a6f338444f0f459ca3403c7a501cc9a2, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/88130f4e637e4847ac02fbab4290b511, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/bedbe9c9b12341258acab2238fbfbb90] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp, totalSize=50.4 K 2024-11-25T03:37:02,606 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/8d4bbebed0324bcd9ddd2e014efec18b, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/5386e50525b24a2d8840f20fad1aa80a, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/bded33507fa549c2bc29fc181238fb8f, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/bb28620f0b45402c814d00b4dfd8e8e9] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp, totalSize=48.0 K 2024-11-25T03:37:02,607 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 343b88cc95db4a36b10c3ae5f0b80fc0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732505819614 2024-11-25T03:37:02,607 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d4bbebed0324bcd9ddd2e014efec18b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732505819614 2024-11-25T03:37:02,607 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6f338444f0f459ca3403c7a501cc9a2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732505820610 2024-11-25T03:37:02,607 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 5386e50525b24a2d8840f20fad1aa80a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732505820610 2024-11-25T03:37:02,608 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88130f4e637e4847ac02fbab4290b511, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732505820657 2024-11-25T03:37:02,608 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting bded33507fa549c2bc29fc181238fb8f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732505820657 2024-11-25T03:37:02,609 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting bedbe9c9b12341258acab2238fbfbb90, keycount=200, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732505821014 2024-11-25T03:37:02,609 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting bb28620f0b45402c814d00b4dfd8e8e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732505821014 2024-11-25T03:37:02,634 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee186277170c2bdfb070a7c209286857#A#compaction#54 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:37:02,635 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/eadb6bdb99e947d1aa7101218f025793 is 50, key is test_row_0/A:col10/1732505821014/Put/seqid=0 2024-11-25T03:37:02,640 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee186277170c2bdfb070a7c209286857#B#compaction#55 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:37:02,641 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/cf3871be4b5143a790b5c145ddbb6f68 is 50, key is test_row_0/B:col10/1732505821014/Put/seqid=0 2024-11-25T03:37:02,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741892_1068 (size=12865) 2024-11-25T03:37:02,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741891_1067 (size=12865) 2024-11-25T03:37:02,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-25T03:37:02,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:02,672 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing ee186277170c2bdfb070a7c209286857 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-25T03:37:02,675 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/cf3871be4b5143a790b5c145ddbb6f68 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/cf3871be4b5143a790b5c145ddbb6f68 2024-11-25T03:37:02,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/1d4a111bf0b045d1ac34cf2d14072ea5 is 50, key is test_row_0/A:col10/1732505821694/Put/seqid=0 2024-11-25T03:37:02,687 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/eadb6bdb99e947d1aa7101218f025793 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/eadb6bdb99e947d1aa7101218f025793 2024-11-25T03:37:02,689 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee186277170c2bdfb070a7c209286857/B of ee186277170c2bdfb070a7c209286857 into cf3871be4b5143a790b5c145ddbb6f68(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:37:02,689 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:37:02,690 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., storeName=ee186277170c2bdfb070a7c209286857/B, priority=12, startTime=1732505822603; duration=0sec 2024-11-25T03:37:02,690 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:37:02,690 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee186277170c2bdfb070a7c209286857:B 2024-11-25T03:37:02,691 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:37:02,698 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49182 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:37:02,698 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): ee186277170c2bdfb070a7c209286857/C is initiating minor compaction (all files) 2024-11-25T03:37:02,698 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee186277170c2bdfb070a7c209286857/C in TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:02,701 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/b1cdc4c3a05f4146844fc959665706b8, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/9b56b55cb36544ddbd3630f1849197e1, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/b8c0eebc0a774bdd9fc502ab35e9f9d2, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/25ead75a50df4ca3bad2d9419428d210] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp, totalSize=48.0 K 2024-11-25T03:37:02,701 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting b1cdc4c3a05f4146844fc959665706b8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732505819614 2024-11-25T03:37:02,702 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b56b55cb36544ddbd3630f1849197e1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732505820610 2024-11-25T03:37:02,702 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting b8c0eebc0a774bdd9fc502ab35e9f9d2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732505820657 2024-11-25T03:37:02,703 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 25ead75a50df4ca3bad2d9419428d210, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732505821014 2024-11-25T03:37:02,709 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee186277170c2bdfb070a7c209286857/A of ee186277170c2bdfb070a7c209286857 into eadb6bdb99e947d1aa7101218f025793(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:37:02,709 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:37:02,709 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., storeName=ee186277170c2bdfb070a7c209286857/A, priority=12, startTime=1732505822602; duration=0sec 2024-11-25T03:37:02,709 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:02,709 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee186277170c2bdfb070a7c209286857:A 2024-11-25T03:37:02,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741893_1069 (size=12301) 2024-11-25T03:37:02,727 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/1d4a111bf0b045d1ac34cf2d14072ea5 2024-11-25T03:37:02,732 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee186277170c2bdfb070a7c209286857#C#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:37:02,732 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/efdd29c235294d5e840eef87e03b46c0 is 50, key is test_row_0/C:col10/1732505821014/Put/seqid=0 2024-11-25T03:37:02,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741894_1070 (size=12865) 2024-11-25T03:37:02,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/d11c828ca88c4102b7296fc77e81110c is 50, key is test_row_0/B:col10/1732505821694/Put/seqid=0 2024-11-25T03:37:02,756 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/efdd29c235294d5e840eef87e03b46c0 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/efdd29c235294d5e840eef87e03b46c0 2024-11-25T03:37:02,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741895_1071 (size=12301) 2024-11-25T03:37:02,774 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/d11c828ca88c4102b7296fc77e81110c 2024-11-25T03:37:02,776 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee186277170c2bdfb070a7c209286857/C of ee186277170c2bdfb070a7c209286857 into efdd29c235294d5e840eef87e03b46c0(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:37:02,776 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:37:02,776 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., storeName=ee186277170c2bdfb070a7c209286857/C, priority=12, startTime=1732505822603; duration=0sec 2024-11-25T03:37:02,777 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:02,777 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee186277170c2bdfb070a7c209286857:C 2024-11-25T03:37:02,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/4053d9dcb7894e10b81e9d90d84105ab is 50, key is test_row_0/C:col10/1732505821694/Put/seqid=0 2024-11-25T03:37:02,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741896_1072 (size=12301) 2024-11-25T03:37:02,800 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/4053d9dcb7894e10b81e9d90d84105ab 2024-11-25T03:37:02,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/1d4a111bf0b045d1ac34cf2d14072ea5 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/1d4a111bf0b045d1ac34cf2d14072ea5 2024-11-25T03:37:02,820 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/1d4a111bf0b045d1ac34cf2d14072ea5, entries=150, sequenceid=284, filesize=12.0 K 2024-11-25T03:37:02,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/d11c828ca88c4102b7296fc77e81110c as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/d11c828ca88c4102b7296fc77e81110c 2024-11-25T03:37:02,830 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/d11c828ca88c4102b7296fc77e81110c, entries=150, sequenceid=284, filesize=12.0 K 2024-11-25T03:37:02,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/4053d9dcb7894e10b81e9d90d84105ab as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/4053d9dcb7894e10b81e9d90d84105ab 2024-11-25T03:37:02,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on ee186277170c2bdfb070a7c209286857 2024-11-25T03:37:02,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. as already flushing 2024-11-25T03:37:02,841 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/4053d9dcb7894e10b81e9d90d84105ab, entries=150, sequenceid=284, filesize=12.0 K 2024-11-25T03:37:02,843 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=6.71 KB/6870 for ee186277170c2bdfb070a7c209286857 in 171ms, sequenceid=284, compaction requested=false 2024-11-25T03:37:02,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:37:02,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:02,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-25T03:37:02,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-25T03:37:02,848 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-25T03:37:02,848 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 952 msec 2024-11-25T03:37:02,852 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 960 msec 2024-11-25T03:37:02,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on ee186277170c2bdfb070a7c209286857 2024-11-25T03:37:02,858 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:37:02,858 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee186277170c2bdfb070a7c209286857 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-25T03:37:02,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/55ea99d3fef445ba9751d5b90eb32cfc is 50, key is test_row_0/A:col10/1732505822849/Put/seqid=0 2024-11-25T03:37:02,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:02,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505882893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:02,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:02,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:02,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:02,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:02,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505882902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:02,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:02,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:02,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:02,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:02,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505882902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:02,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:02,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:02,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:02,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:02,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505882904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:02,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:02,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:02,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:02,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:02,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505882906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:02,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:02,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:02,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:02,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741897_1073 (size=12301) 2024-11-25T03:37:02,928 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/55ea99d3fef445ba9751d5b90eb32cfc 2024-11-25T03:37:02,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/5ee84ec1183a4a28881e7a06fe28a4e4 is 50, key is test_row_0/B:col10/1732505822849/Put/seqid=0 2024-11-25T03:37:02,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741898_1074 (size=12301) 2024-11-25T03:37:02,978 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/5ee84ec1183a4a28881e7a06fe28a4e4 2024-11-25T03:37:02,989 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/c1aafcdcc21344cba885d609b182343f is 50, key is test_row_0/C:col10/1732505822849/Put/seqid=0 2024-11-25T03:37:03,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741899_1075 (size=12301) 2024-11-25T03:37:03,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:03,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505883017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:03,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:03,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:03,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:03,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505883017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:03,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:03,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:03,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505883018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:03,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:03,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:03,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:03,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:03,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:03,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:03,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:03,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505883018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:03,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:03,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:03,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:03,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:03,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505883018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:03,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:03,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:03,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:03,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-25T03:37:03,027 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:37:03,028 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:37:03,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-25T03:37:03,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-25T03:37:03,031 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:37:03,032 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:37:03,032 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:37:03,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-25T03:37:03,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-25T03:37:03,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:03,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. as already flushing 2024-11-25T03:37:03,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:03,186 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:03,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:03,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:03,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:03,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505883227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:03,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:03,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:03,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:03,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:03,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505883227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:03,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:03,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:03,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:03,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:03,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505883227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:03,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:03,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:03,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:03,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:03,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505883228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:03,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:03,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:03,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:03,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:03,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505883228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:03,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:03,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:03,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:03,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-25T03:37:03,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:03,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. as already flushing 2024-11-25T03:37:03,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:03,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:03,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:03,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:03,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-25T03:37:03,403 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/c1aafcdcc21344cba885d609b182343f 2024-11-25T03:37:03,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/55ea99d3fef445ba9751d5b90eb32cfc as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/55ea99d3fef445ba9751d5b90eb32cfc 2024-11-25T03:37:03,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/55ea99d3fef445ba9751d5b90eb32cfc, entries=150, sequenceid=302, filesize=12.0 K 2024-11-25T03:37:03,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/5ee84ec1183a4a28881e7a06fe28a4e4 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/5ee84ec1183a4a28881e7a06fe28a4e4 2024-11-25T03:37:03,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/5ee84ec1183a4a28881e7a06fe28a4e4, entries=150, sequenceid=302, filesize=12.0 K 2024-11-25T03:37:03,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/c1aafcdcc21344cba885d609b182343f as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/c1aafcdcc21344cba885d609b182343f 2024-11-25T03:37:03,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/c1aafcdcc21344cba885d609b182343f, entries=150, sequenceid=302, filesize=12.0 K 2024-11-25T03:37:03,436 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for ee186277170c2bdfb070a7c209286857 in 578ms, sequenceid=302, compaction requested=true 2024-11-25T03:37:03,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:37:03,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee186277170c2bdfb070a7c209286857:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:37:03,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:03,437 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:37:03,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee186277170c2bdfb070a7c209286857:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:37:03,437 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:37:03,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:03,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee186277170c2bdfb070a7c209286857:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:37:03,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:37:03,438 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37467 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:37:03,438 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37467 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:37:03,438 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): ee186277170c2bdfb070a7c209286857/A is initiating minor compaction (all files) 2024-11-25T03:37:03,438 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): ee186277170c2bdfb070a7c209286857/B is initiating minor compaction (all files) 2024-11-25T03:37:03,438 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee186277170c2bdfb070a7c209286857/A in TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:03,438 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee186277170c2bdfb070a7c209286857/B in TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:03,439 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/eadb6bdb99e947d1aa7101218f025793, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/1d4a111bf0b045d1ac34cf2d14072ea5, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/55ea99d3fef445ba9751d5b90eb32cfc] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp, totalSize=36.6 K 2024-11-25T03:37:03,439 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/cf3871be4b5143a790b5c145ddbb6f68, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/d11c828ca88c4102b7296fc77e81110c, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/5ee84ec1183a4a28881e7a06fe28a4e4] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp, totalSize=36.6 K 2024-11-25T03:37:03,439 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting eadb6bdb99e947d1aa7101218f025793, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732505821014 2024-11-25T03:37:03,439 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting cf3871be4b5143a790b5c145ddbb6f68, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732505821014 2024-11-25T03:37:03,440 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d4a111bf0b045d1ac34cf2d14072ea5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732505821693 2024-11-25T03:37:03,440 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting d11c828ca88c4102b7296fc77e81110c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732505821693 2024-11-25T03:37:03,440 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55ea99d3fef445ba9751d5b90eb32cfc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732505822849 2024-11-25T03:37:03,441 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ee84ec1183a4a28881e7a06fe28a4e4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732505822849 2024-11-25T03:37:03,452 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee186277170c2bdfb070a7c209286857#B#compaction#63 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:37:03,453 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee186277170c2bdfb070a7c209286857#A#compaction#64 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:37:03,453 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/b0b1b53635c644199191855c06e93e05 is 50, key is test_row_0/B:col10/1732505822849/Put/seqid=0 2024-11-25T03:37:03,453 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/8a2954eabeb94f54a9371b30bc7244f9 is 50, key is test_row_0/A:col10/1732505822849/Put/seqid=0 2024-11-25T03:37:03,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741900_1076 (size=13017) 2024-11-25T03:37:03,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741901_1077 (size=13017) 2024-11-25T03:37:03,481 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/b0b1b53635c644199191855c06e93e05 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/b0b1b53635c644199191855c06e93e05 2024-11-25T03:37:03,490 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee186277170c2bdfb070a7c209286857/B of ee186277170c2bdfb070a7c209286857 into b0b1b53635c644199191855c06e93e05(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:37:03,490 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:37:03,490 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., storeName=ee186277170c2bdfb070a7c209286857/B, priority=13, startTime=1732505823437; duration=0sec 2024-11-25T03:37:03,491 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:37:03,491 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee186277170c2bdfb070a7c209286857:B 2024-11-25T03:37:03,491 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:37:03,492 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37467 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:37:03,492 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): ee186277170c2bdfb070a7c209286857/C is initiating minor compaction (all files) 2024-11-25T03:37:03,492 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee186277170c2bdfb070a7c209286857/C in TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:03,493 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/efdd29c235294d5e840eef87e03b46c0, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/4053d9dcb7894e10b81e9d90d84105ab, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/c1aafcdcc21344cba885d609b182343f] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp, totalSize=36.6 K 2024-11-25T03:37:03,493 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting efdd29c235294d5e840eef87e03b46c0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732505821014 2024-11-25T03:37:03,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-25T03:37:03,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:03,494 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 4053d9dcb7894e10b81e9d90d84105ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732505821693 2024-11-25T03:37:03,494 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing ee186277170c2bdfb070a7c209286857 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-25T03:37:03,494 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting c1aafcdcc21344cba885d609b182343f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732505822849 2024-11-25T03:37:03,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/b7bd8e46c82e44e4889492c77808ec26 is 50, key is test_row_0/A:col10/1732505822893/Put/seqid=0 2024-11-25T03:37:03,512 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee186277170c2bdfb070a7c209286857#C#compaction#66 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:37:03,513 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/6aeea4ea4e1e470294f5ae9068f306cc is 50, key is test_row_0/C:col10/1732505822849/Put/seqid=0 2024-11-25T03:37:03,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741902_1078 (size=12301) 2024-11-25T03:37:03,519 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/b7bd8e46c82e44e4889492c77808ec26 2024-11-25T03:37:03,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/fa56a78b8b534c1eafc7f6fc57e64537 is 50, key is test_row_0/B:col10/1732505822893/Put/seqid=0 2024-11-25T03:37:03,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741903_1079 (size=13017) 2024-11-25T03:37:03,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on ee186277170c2bdfb070a7c209286857 2024-11-25T03:37:03,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. as already flushing 2024-11-25T03:37:03,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741904_1080 (size=12301) 2024-11-25T03:37:03,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:03,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505883562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:03,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:03,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:03,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505883563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:03,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:03,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505883563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:03,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:03,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:03,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:03,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:03,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:03,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:03,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505883565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:03,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:03,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:03,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:03,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:03,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:03,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505883565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:03,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:03,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:03,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:03,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:03,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:03,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-25T03:37:03,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:03,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505883677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:03,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:03,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:03,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:03,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:03,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505883678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:03,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:03,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505883678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:03,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:03,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505883678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:03,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:03,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505883678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:03,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:03,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:03,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:03,684 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:03,684 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:03,684 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:03,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:03,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:03,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:03,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:03,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:03,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:03,886 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/8a2954eabeb94f54a9371b30bc7244f9 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/8a2954eabeb94f54a9371b30bc7244f9 2024-11-25T03:37:03,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:03,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:03,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505883888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:03,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505883887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:03,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:03,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:03,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505883888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:03,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505883888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:03,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:03,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:03,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:03,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:03,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:03,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:03,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:03,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505883898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:03,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:03,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:03,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:03,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:03,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:03,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:03,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:03,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:03,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:03,909 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee186277170c2bdfb070a7c209286857/A of ee186277170c2bdfb070a7c209286857 into 8a2954eabeb94f54a9371b30bc7244f9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:37:03,909 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:37:03,909 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., storeName=ee186277170c2bdfb070a7c209286857/A, priority=13, startTime=1732505823437; duration=0sec 2024-11-25T03:37:03,909 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:03,909 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee186277170c2bdfb070a7c209286857:A 2024-11-25T03:37:03,949 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/6aeea4ea4e1e470294f5ae9068f306cc as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/6aeea4ea4e1e470294f5ae9068f306cc 2024-11-25T03:37:03,951 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/fa56a78b8b534c1eafc7f6fc57e64537 2024-11-25T03:37:03,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/36f073135c5943eeb2dbcbdb367eb28d is 50, key is test_row_0/C:col10/1732505822893/Put/seqid=0 2024-11-25T03:37:03,969 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee186277170c2bdfb070a7c209286857/C of ee186277170c2bdfb070a7c209286857 into 6aeea4ea4e1e470294f5ae9068f306cc(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:37:03,969 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:37:03,969 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., storeName=ee186277170c2bdfb070a7c209286857/C, priority=13, startTime=1732505823437; duration=0sec 2024-11-25T03:37:03,970 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:03,970 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee186277170c2bdfb070a7c209286857:C 2024-11-25T03:37:03,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741905_1081 (size=12301) 2024-11-25T03:37:04,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-25T03:37:04,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:04,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505884197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:04,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:04,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:04,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:04,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:04,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505884208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:04,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:04,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:04,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:04,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505884208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:04,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:04,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:04,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:04,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505884208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:04,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:04,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:04,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:04,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505884208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:04,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:04,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:04,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:04,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:04,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:04,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:04,387 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/36f073135c5943eeb2dbcbdb367eb28d 2024-11-25T03:37:04,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/b7bd8e46c82e44e4889492c77808ec26 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/b7bd8e46c82e44e4889492c77808ec26 2024-11-25T03:37:04,406 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/b7bd8e46c82e44e4889492c77808ec26, entries=150, sequenceid=324, filesize=12.0 K 2024-11-25T03:37:04,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/fa56a78b8b534c1eafc7f6fc57e64537 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/fa56a78b8b534c1eafc7f6fc57e64537 2024-11-25T03:37:04,424 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/fa56a78b8b534c1eafc7f6fc57e64537, entries=150, sequenceid=324, filesize=12.0 K 2024-11-25T03:37:04,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/36f073135c5943eeb2dbcbdb367eb28d as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/36f073135c5943eeb2dbcbdb367eb28d 2024-11-25T03:37:04,433 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/36f073135c5943eeb2dbcbdb367eb28d, entries=150, sequenceid=324, filesize=12.0 K 2024-11-25T03:37:04,440 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for ee186277170c2bdfb070a7c209286857 in 946ms, sequenceid=324, compaction requested=false 2024-11-25T03:37:04,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:37:04,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:04,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-25T03:37:04,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-25T03:37:04,445 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-25T03:37:04,445 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4110 sec 2024-11-25T03:37:04,450 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.4190 sec 2024-11-25T03:37:04,712 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:37:04,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on ee186277170c2bdfb070a7c209286857 2024-11-25T03:37:04,712 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee186277170c2bdfb070a7c209286857 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-25T03:37:04,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/5fc6ff77e2ad4dc2bd5af8c06c608734 is 50, key is test_row_0/A:col10/1732505823562/Put/seqid=0 2024-11-25T03:37:04,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:04,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505884737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:04,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:04,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505884739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:04,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:04,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:04,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:04,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:04,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505884740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:04,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:04,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:04,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:04,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:04,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:04,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:04,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741906_1082 (size=12301) 2024-11-25T03:37:04,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:04,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505884744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:04,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:04,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:04,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:04,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:04,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505884746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:04,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:04,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:04,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:04,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:04,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505884857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:04,861 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:04,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:04,861 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:04,861 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:04,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505884857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:04,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:04,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:04,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:04,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505884858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:04,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:04,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:04,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505884858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:04,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:04,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:04,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:04,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:04,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:04,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:04,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505884858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:04,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:04,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:04,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:04,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:05,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:05,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505885067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:05,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:05,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505885067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:05,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:05,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:05,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:05,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:05,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:05,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:05,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:05,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:05,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505885068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:05,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505885068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:05,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:05,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:05,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505885068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:05,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:05,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:05,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:05,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:05,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:05,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:05,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:05,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:05,151 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/5fc6ff77e2ad4dc2bd5af8c06c608734 2024-11-25T03:37:05,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/557bc182dad544b4b44f0c4ec8e0b5b5 is 50, key is test_row_0/B:col10/1732505823562/Put/seqid=0 2024-11-25T03:37:05,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741907_1083 (size=12301) 2024-11-25T03:37:05,171 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/557bc182dad544b4b44f0c4ec8e0b5b5 2024-11-25T03:37:05,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-25T03:37:05,179 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:37:05,179 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:37:05,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-25T03:37:05,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-25T03:37:05,183 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:37:05,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/6759dc9d7971404196ee7e1ce0aee090 is 50, key is test_row_0/C:col10/1732505823562/Put/seqid=0 2024-11-25T03:37:05,185 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:37:05,186 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:37:05,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741908_1084 (size=12301) 2024-11-25T03:37:05,201 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/6759dc9d7971404196ee7e1ce0aee090 2024-11-25T03:37:05,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/5fc6ff77e2ad4dc2bd5af8c06c608734 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/5fc6ff77e2ad4dc2bd5af8c06c608734 2024-11-25T03:37:05,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/5fc6ff77e2ad4dc2bd5af8c06c608734, entries=150, sequenceid=343, filesize=12.0 K 2024-11-25T03:37:05,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/557bc182dad544b4b44f0c4ec8e0b5b5 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/557bc182dad544b4b44f0c4ec8e0b5b5 2024-11-25T03:37:05,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/557bc182dad544b4b44f0c4ec8e0b5b5, entries=150, sequenceid=343, filesize=12.0 K 2024-11-25T03:37:05,229 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-25T03:37:05,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/6759dc9d7971404196ee7e1ce0aee090 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/6759dc9d7971404196ee7e1ce0aee090 2024-11-25T03:37:05,239 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/6759dc9d7971404196ee7e1ce0aee090, entries=150, sequenceid=343, filesize=12.0 K 2024-11-25T03:37:05,241 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for ee186277170c2bdfb070a7c209286857 in 529ms, sequenceid=343, compaction requested=true 2024-11-25T03:37:05,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:37:05,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee186277170c2bdfb070a7c209286857:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:37:05,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:05,241 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:37:05,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee186277170c2bdfb070a7c209286857:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:37:05,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:05,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee186277170c2bdfb070a7c209286857:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:37:05,241 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:37:05,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:37:05,243 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:37:05,243 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): ee186277170c2bdfb070a7c209286857/A is initiating minor compaction (all files) 2024-11-25T03:37:05,243 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee186277170c2bdfb070a7c209286857/A in TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:05,243 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/8a2954eabeb94f54a9371b30bc7244f9, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/b7bd8e46c82e44e4889492c77808ec26, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/5fc6ff77e2ad4dc2bd5af8c06c608734] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp, totalSize=36.7 K 2024-11-25T03:37:05,244 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:37:05,244 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a2954eabeb94f54a9371b30bc7244f9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732505822849 2024-11-25T03:37:05,244 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): ee186277170c2bdfb070a7c209286857/B is initiating minor compaction (all files) 2024-11-25T03:37:05,244 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee186277170c2bdfb070a7c209286857/B in TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:05,244 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/b0b1b53635c644199191855c06e93e05, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/fa56a78b8b534c1eafc7f6fc57e64537, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/557bc182dad544b4b44f0c4ec8e0b5b5] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp, totalSize=36.7 K 2024-11-25T03:37:05,244 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7bd8e46c82e44e4889492c77808ec26, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732505822893 2024-11-25T03:37:05,245 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting b0b1b53635c644199191855c06e93e05, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732505822849 2024-11-25T03:37:05,245 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fc6ff77e2ad4dc2bd5af8c06c608734, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732505823559 2024-11-25T03:37:05,246 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting fa56a78b8b534c1eafc7f6fc57e64537, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732505822893 2024-11-25T03:37:05,247 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 557bc182dad544b4b44f0c4ec8e0b5b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732505823559 2024-11-25T03:37:05,263 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee186277170c2bdfb070a7c209286857#A#compaction#72 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:37:05,264 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/f4eb4f81c84e4fc58b7b5afbc270aa99 is 50, key is test_row_0/A:col10/1732505823562/Put/seqid=0 2024-11-25T03:37:05,271 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee186277170c2bdfb070a7c209286857#B#compaction#73 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:37:05,271 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/3e190c38e4584194aeff476a7fa64f7d is 50, key is test_row_0/B:col10/1732505823562/Put/seqid=0 2024-11-25T03:37:05,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-25T03:37:05,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741909_1085 (size=13119) 2024-11-25T03:37:05,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741910_1086 (size=13119) 2024-11-25T03:37:05,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-25T03:37:05,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:05,341 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing ee186277170c2bdfb070a7c209286857 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-25T03:37:05,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/75389c1961f844dfad5c8bb8d5f7c9c0 is 50, key is test_row_0/A:col10/1732505824741/Put/seqid=0 2024-11-25T03:37:05,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on ee186277170c2bdfb070a7c209286857 2024-11-25T03:37:05,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. as already flushing 2024-11-25T03:37:05,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741911_1087 (size=12301) 2024-11-25T03:37:05,399 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/75389c1961f844dfad5c8bb8d5f7c9c0 2024-11-25T03:37:05,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/f7ff0625629b49ada76f03a080ece6db is 50, key is test_row_0/B:col10/1732505824741/Put/seqid=0 2024-11-25T03:37:05,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741912_1088 (size=12301) 2024-11-25T03:37:05,433 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/f7ff0625629b49ada76f03a080ece6db 2024-11-25T03:37:05,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/4e2b41ce677f4ae1b40f499987a89e6f is 50, key is test_row_0/C:col10/1732505824741/Put/seqid=0 2024-11-25T03:37:05,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:05,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:05,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505885466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:05,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505885466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:05,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:05,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:05,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:05,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:05,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:05,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:05,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:05,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505885472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:05,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:05,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:05,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:05,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:05,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505885473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:05,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:05,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:05,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:05,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:05,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505885473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:05,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:05,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:05,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:05,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741913_1089 (size=12301) 2024-11-25T03:37:05,499 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/4e2b41ce677f4ae1b40f499987a89e6f 2024-11-25T03:37:05,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-25T03:37:05,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/75389c1961f844dfad5c8bb8d5f7c9c0 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/75389c1961f844dfad5c8bb8d5f7c9c0 2024-11-25T03:37:05,519 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/75389c1961f844dfad5c8bb8d5f7c9c0, entries=150, sequenceid=362, filesize=12.0 K 2024-11-25T03:37:05,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/f7ff0625629b49ada76f03a080ece6db as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/f7ff0625629b49ada76f03a080ece6db 2024-11-25T03:37:05,527 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/f7ff0625629b49ada76f03a080ece6db, entries=150, sequenceid=362, filesize=12.0 K 2024-11-25T03:37:05,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/4e2b41ce677f4ae1b40f499987a89e6f as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/4e2b41ce677f4ae1b40f499987a89e6f 2024-11-25T03:37:05,537 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/4e2b41ce677f4ae1b40f499987a89e6f, entries=150, sequenceid=362, filesize=12.0 K 2024-11-25T03:37:05,540 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=107.34 KB/109920 for ee186277170c2bdfb070a7c209286857 in 200ms, sequenceid=362, compaction requested=true 2024-11-25T03:37:05,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:37:05,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:05,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-25T03:37:05,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-25T03:37:05,548 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-25T03:37:05,548 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 361 msec 2024-11-25T03:37:05,553 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 372 msec 2024-11-25T03:37:05,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on ee186277170c2bdfb070a7c209286857 2024-11-25T03:37:05,580 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:37:05,580 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee186277170c2bdfb070a7c209286857 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-25T03:37:05,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/598260aa127c4d208a30e10c193a924b is 50, key is test_row_0/A:col10/1732505825396/Put/seqid=0 2024-11-25T03:37:05,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:05,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505885604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:05,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:05,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:05,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505885606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:05,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:05,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:05,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:05,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:05,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505885607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:05,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:05,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:05,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:05,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:05,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:05,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505885609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:05,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:05,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:05,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:05,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:05,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:05,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505885614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:05,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:05,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:05,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:05,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741914_1090 (size=14741) 2024-11-25T03:37:05,710 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/f4eb4f81c84e4fc58b7b5afbc270aa99 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/f4eb4f81c84e4fc58b7b5afbc270aa99 2024-11-25T03:37:05,720 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/3e190c38e4584194aeff476a7fa64f7d as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/3e190c38e4584194aeff476a7fa64f7d 2024-11-25T03:37:05,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:05,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505885717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:05,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:05,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:05,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:05,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:05,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505885718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:05,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:05,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:05,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:05,726 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee186277170c2bdfb070a7c209286857/A of ee186277170c2bdfb070a7c209286857 into f4eb4f81c84e4fc58b7b5afbc270aa99(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:37:05,726 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:37:05,726 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., storeName=ee186277170c2bdfb070a7c209286857/A, priority=13, startTime=1732505825241; duration=0sec 2024-11-25T03:37:05,726 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:37:05,726 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee186277170c2bdfb070a7c209286857:A 2024-11-25T03:37:05,727 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:37:05,730 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:37:05,731 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): ee186277170c2bdfb070a7c209286857/C is initiating minor compaction (all files) 2024-11-25T03:37:05,731 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee186277170c2bdfb070a7c209286857/C in TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:05,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:05,731 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/6aeea4ea4e1e470294f5ae9068f306cc, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/36f073135c5943eeb2dbcbdb367eb28d, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/6759dc9d7971404196ee7e1ce0aee090, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/4e2b41ce677f4ae1b40f499987a89e6f] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp, totalSize=48.8 K 2024-11-25T03:37:05,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505885728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:05,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:05,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:05,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:05,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:05,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505885728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:05,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:05,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:05,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:05,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:05,732 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6aeea4ea4e1e470294f5ae9068f306cc, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732505822849 2024-11-25T03:37:05,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505885728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:05,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:05,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:05,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:05,733 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36f073135c5943eeb2dbcbdb367eb28d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732505822893 2024-11-25T03:37:05,733 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee186277170c2bdfb070a7c209286857/B of ee186277170c2bdfb070a7c209286857 into 3e190c38e4584194aeff476a7fa64f7d(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:37:05,733 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:37:05,733 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., storeName=ee186277170c2bdfb070a7c209286857/B, priority=13, startTime=1732505825241; duration=0sec 2024-11-25T03:37:05,733 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:05,733 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee186277170c2bdfb070a7c209286857:B 2024-11-25T03:37:05,734 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6759dc9d7971404196ee7e1ce0aee090, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732505823559 2024-11-25T03:37:05,734 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e2b41ce677f4ae1b40f499987a89e6f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1732505824738 2024-11-25T03:37:05,753 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee186277170c2bdfb070a7c209286857#C#compaction#78 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:37:05,754 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/a6c3686fd0e64a5c8b54613daafef776 is 50, key is test_row_0/C:col10/1732505824741/Put/seqid=0 2024-11-25T03:37:05,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741915_1091 (size=13153) 2024-11-25T03:37:05,794 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/a6c3686fd0e64a5c8b54613daafef776 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/a6c3686fd0e64a5c8b54613daafef776 2024-11-25T03:37:05,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-25T03:37:05,810 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:37:05,810 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee186277170c2bdfb070a7c209286857/C of ee186277170c2bdfb070a7c209286857 into a6c3686fd0e64a5c8b54613daafef776(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:37:05,810 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:37:05,810 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., storeName=ee186277170c2bdfb070a7c209286857/C, priority=12, startTime=1732505825241; duration=0sec 2024-11-25T03:37:05,811 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:05,811 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee186277170c2bdfb070a7c209286857:C 2024-11-25T03:37:05,811 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:37:05,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-25T03:37:05,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-25T03:37:05,815 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:37:05,816 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:37:05,816 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:37:05,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-25T03:37:05,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:05,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505885927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:05,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:05,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505885927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:05,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:05,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:05,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:05,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:05,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:05,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:05,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:05,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505885938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:05,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:05,943 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:05,943 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:05,943 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:05,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505885938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:05,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:05,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505885938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:05,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:05,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:05,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:05,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:05,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:05,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:05,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-25T03:37:05,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:05,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. as already flushing 2024-11-25T03:37:05,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:05,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:05,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:05,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:06,035 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/598260aa127c4d208a30e10c193a924b 2024-11-25T03:37:06,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/213a1cc75d86424f84cba3390ede6a82 is 50, key is test_row_0/B:col10/1732505825396/Put/seqid=0 2024-11-25T03:37:06,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741916_1092 (size=12301) 2024-11-25T03:37:06,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-25T03:37:06,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-25T03:37:06,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:06,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. as already flushing 2024-11-25T03:37:06,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:06,129 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:06,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:06,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:06,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:06,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505886247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:06,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:06,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:06,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:06,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:06,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505886248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:06,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:06,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:06,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:06,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:06,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505886248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:06,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:06,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:06,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:06,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:06,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505886257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:06,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:06,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505886257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:06,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:06,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:06,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:06,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:06,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:06,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:06,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-25T03:37:06,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:06,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. as already flushing 2024-11-25T03:37:06,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:06,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:06,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:06,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:06,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-25T03:37:06,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:06,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. as already flushing 2024-11-25T03:37:06,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:06,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:06,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:06,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-25T03:37:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:06,472 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/213a1cc75d86424f84cba3390ede6a82 2024-11-25T03:37:06,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/0081d65d2a1545a29fad9060968af23b is 50, key is test_row_0/C:col10/1732505825396/Put/seqid=0 2024-11-25T03:37:06,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741917_1093 (size=12301) 2024-11-25T03:37:06,513 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/0081d65d2a1545a29fad9060968af23b 2024-11-25T03:37:06,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/598260aa127c4d208a30e10c193a924b as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/598260aa127c4d208a30e10c193a924b 2024-11-25T03:37:06,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/598260aa127c4d208a30e10c193a924b, entries=200, sequenceid=382, filesize=14.4 K 2024-11-25T03:37:06,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/213a1cc75d86424f84cba3390ede6a82 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/213a1cc75d86424f84cba3390ede6a82 2024-11-25T03:37:06,539 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/213a1cc75d86424f84cba3390ede6a82, entries=150, sequenceid=382, filesize=12.0 K 2024-11-25T03:37:06,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/0081d65d2a1545a29fad9060968af23b as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/0081d65d2a1545a29fad9060968af23b 2024-11-25T03:37:06,548 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/0081d65d2a1545a29fad9060968af23b, entries=150, sequenceid=382, filesize=12.0 K 2024-11-25T03:37:06,549 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for ee186277170c2bdfb070a7c209286857 in 969ms, sequenceid=382, compaction requested=true 2024-11-25T03:37:06,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:37:06,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee186277170c2bdfb070a7c209286857:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:37:06,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:06,550 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:37:06,550 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:37:06,551 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:37:06,551 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:37:06,552 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): ee186277170c2bdfb070a7c209286857/B is initiating minor compaction (all files) 2024-11-25T03:37:06,552 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): ee186277170c2bdfb070a7c209286857/A is initiating minor compaction (all files) 2024-11-25T03:37:06,552 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee186277170c2bdfb070a7c209286857/B in TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:06,552 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee186277170c2bdfb070a7c209286857/A in TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:06,552 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/3e190c38e4584194aeff476a7fa64f7d, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/f7ff0625629b49ada76f03a080ece6db, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/213a1cc75d86424f84cba3390ede6a82] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp, totalSize=36.8 K 2024-11-25T03:37:06,552 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/f4eb4f81c84e4fc58b7b5afbc270aa99, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/75389c1961f844dfad5c8bb8d5f7c9c0, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/598260aa127c4d208a30e10c193a924b] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp, totalSize=39.2 K 2024-11-25T03:37:06,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee186277170c2bdfb070a7c209286857:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:37:06,553 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4eb4f81c84e4fc58b7b5afbc270aa99, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732505823559 2024-11-25T03:37:06,553 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e190c38e4584194aeff476a7fa64f7d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732505823559 2024-11-25T03:37:06,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:06,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee186277170c2bdfb070a7c209286857:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:37:06,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:37:06,553 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75389c1961f844dfad5c8bb8d5f7c9c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1732505824738 2024-11-25T03:37:06,555 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting f7ff0625629b49ada76f03a080ece6db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1732505824738 2024-11-25T03:37:06,555 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 598260aa127c4d208a30e10c193a924b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1732505825396 2024-11-25T03:37:06,555 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 213a1cc75d86424f84cba3390ede6a82, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1732505825396 2024-11-25T03:37:06,576 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee186277170c2bdfb070a7c209286857#A#compaction#81 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:37:06,577 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/ef7b5767eabd487ea1d47f718af9ef9b is 50, key is test_row_0/A:col10/1732505825396/Put/seqid=0 2024-11-25T03:37:06,582 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee186277170c2bdfb070a7c209286857#B#compaction#82 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:37:06,583 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/8ff0a55df2ad4ff49ffb61591260b93d is 50, key is test_row_0/B:col10/1732505825396/Put/seqid=0 2024-11-25T03:37:06,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-25T03:37:06,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:06,592 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing ee186277170c2bdfb070a7c209286857 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-25T03:37:06,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741919_1095 (size=13221) 2024-11-25T03:37:06,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741918_1094 (size=13221) 2024-11-25T03:37:06,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/3a466108ba57487a813214b8bb4ac583 is 50, key is test_row_0/A:col10/1732505825604/Put/seqid=0 2024-11-25T03:37:06,629 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/ef7b5767eabd487ea1d47f718af9ef9b as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/ef7b5767eabd487ea1d47f718af9ef9b 2024-11-25T03:37:06,639 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee186277170c2bdfb070a7c209286857/A of ee186277170c2bdfb070a7c209286857 into ef7b5767eabd487ea1d47f718af9ef9b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:37:06,639 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:37:06,639 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., storeName=ee186277170c2bdfb070a7c209286857/A, priority=13, startTime=1732505826550; duration=0sec 2024-11-25T03:37:06,641 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:37:06,641 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee186277170c2bdfb070a7c209286857:A 2024-11-25T03:37:06,641 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-25T03:37:06,642 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-25T03:37:06,643 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-25T03:37:06,643 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. because compaction request was cancelled 2024-11-25T03:37:06,643 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee186277170c2bdfb070a7c209286857:C 2024-11-25T03:37:06,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741920_1096 (size=12301) 2024-11-25T03:37:06,662 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/3a466108ba57487a813214b8bb4ac583 2024-11-25T03:37:06,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/6dc568a4ad3b46b0888283e5faad0b62 is 50, key is test_row_0/B:col10/1732505825604/Put/seqid=0 2024-11-25T03:37:06,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741921_1097 (size=12301) 2024-11-25T03:37:06,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on ee186277170c2bdfb070a7c209286857 2024-11-25T03:37:06,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. as already flushing 2024-11-25T03:37:06,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:06,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505886831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:06,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:06,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505886831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:06,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:06,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:06,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:06,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:06,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505886832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:06,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:06,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:06,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:06,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:06,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505886833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:06,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:06,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:06,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:06,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:06,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:06,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:06,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:06,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505886835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:06,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:06,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:06,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:06,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-25T03:37:06,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:06,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505886948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:06,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:06,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:06,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:06,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505886948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:06,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:06,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:06,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505886948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:06,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:06,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:06,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:06,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505886948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:06,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:06,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505886948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:06,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:06,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:06,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:06,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:06,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:06,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:06,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:06,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:06,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:06,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:07,033 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/8ff0a55df2ad4ff49ffb61591260b93d as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/8ff0a55df2ad4ff49ffb61591260b93d 2024-11-25T03:37:07,041 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee186277170c2bdfb070a7c209286857/B of ee186277170c2bdfb070a7c209286857 into 8ff0a55df2ad4ff49ffb61591260b93d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:37:07,041 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:37:07,041 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., storeName=ee186277170c2bdfb070a7c209286857/B, priority=13, startTime=1732505826550; duration=0sec 2024-11-25T03:37:07,042 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:07,042 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee186277170c2bdfb070a7c209286857:B 2024-11-25T03:37:07,110 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/6dc568a4ad3b46b0888283e5faad0b62 2024-11-25T03:37:07,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/6b7133ab3f214cf9ba9c0caf9b24779d is 50, key is test_row_0/C:col10/1732505825604/Put/seqid=0 2024-11-25T03:37:07,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741922_1098 (size=12301) 2024-11-25T03:37:07,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:07,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505887158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:07,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:07,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:07,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:07,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:07,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505887158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:07,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:07,160 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:07,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505887158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:07,160 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:07,160 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:07,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:07,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:07,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505887158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:07,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:07,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:07,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:07,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505887158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:07,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:07,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:07,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:07,161 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:07,161 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:07,161 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:07,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:07,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:07,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505887467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:07,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505887467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:07,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:07,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:07,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:07,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:07,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505887467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:07,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:07,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505887468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:07,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:07,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:07,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505887468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:07,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:07,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:07,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:07,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:07,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:07,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:07,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:07,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:07,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:07,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:07,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:07,531 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/6b7133ab3f214cf9ba9c0caf9b24779d 2024-11-25T03:37:07,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/3a466108ba57487a813214b8bb4ac583 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/3a466108ba57487a813214b8bb4ac583 2024-11-25T03:37:07,550 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/3a466108ba57487a813214b8bb4ac583, entries=150, sequenceid=401, filesize=12.0 K 2024-11-25T03:37:07,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/6dc568a4ad3b46b0888283e5faad0b62 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/6dc568a4ad3b46b0888283e5faad0b62 2024-11-25T03:37:07,560 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/6dc568a4ad3b46b0888283e5faad0b62, entries=150, sequenceid=401, filesize=12.0 K 2024-11-25T03:37:07,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/6b7133ab3f214cf9ba9c0caf9b24779d as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/6b7133ab3f214cf9ba9c0caf9b24779d 2024-11-25T03:37:07,572 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/6b7133ab3f214cf9ba9c0caf9b24779d, entries=150, sequenceid=401, filesize=12.0 K 2024-11-25T03:37:07,573 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for ee186277170c2bdfb070a7c209286857 in 981ms, sequenceid=401, compaction requested=true 2024-11-25T03:37:07,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:37:07,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:07,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-25T03:37:07,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-25T03:37:07,578 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-25T03:37:07,578 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7590 sec 2024-11-25T03:37:07,581 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.7680 sec 2024-11-25T03:37:07,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-25T03:37:07,957 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:37:07,958 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:37:07,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-25T03:37:07,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-25T03:37:07,961 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:37:07,961 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:37:07,962 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:37:07,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on ee186277170c2bdfb070a7c209286857 2024-11-25T03:37:07,981 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:37:07,981 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee186277170c2bdfb070a7c209286857 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-25T03:37:07,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/4eda90d9af6c485b9c89ba8f5a2576c9 is 50, key is test_row_0/A:col10/1732505826830/Put/seqid=0 2024-11-25T03:37:08,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:08,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505888001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:08,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:08,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505888001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:08,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:08,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:08,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741923_1099 (size=17181) 2024-11-25T03:37:08,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:08,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505888004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:08,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:08,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:08,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505888004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:08,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:08,013 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/4eda90d9af6c485b9c89ba8f5a2576c9 2024-11-25T03:37:08,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:08,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505888012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:08,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:08,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/02ed19764cfe41ff8482396eaf764b9e is 50, key is test_row_0/B:col10/1732505826830/Put/seqid=0 2024-11-25T03:37:08,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741924_1100 (size=12301) 2024-11-25T03:37:08,048 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/02ed19764cfe41ff8482396eaf764b9e 2024-11-25T03:37:08,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/acfe6dba1eb8466dbc8a29eb0f263458 is 50, key is test_row_0/C:col10/1732505826830/Put/seqid=0 2024-11-25T03:37:08,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741925_1101 (size=12301) 2024-11-25T03:37:08,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-25T03:37:08,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-25T03:37:08,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:08,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. as already flushing 2024-11-25T03:37:08,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:08,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:08,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:08,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:08,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:08,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:08,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505888117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:08,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505888118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:08,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:08,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505888118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:08,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:08,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:08,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:08,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:08,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505888118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:08,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:08,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505888127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:08,128 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,128 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,128 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:08,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:08,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-25T03:37:08,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:08,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. as already flushing 2024-11-25T03:37:08,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:08,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:08,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:08,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:08,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-25T03:37:08,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:08,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505888327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:08,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:08,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:08,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505888327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:08,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:08,330 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,330 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505888327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:08,330 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:08,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:08,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:08,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505888337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:08,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:08,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:08,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505888337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:08,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:08,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-25T03:37:08,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:08,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. as already flushing 2024-11-25T03:37:08,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:08,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:08,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:08,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:08,464 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/acfe6dba1eb8466dbc8a29eb0f263458 2024-11-25T03:37:08,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/4eda90d9af6c485b9c89ba8f5a2576c9 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/4eda90d9af6c485b9c89ba8f5a2576c9 2024-11-25T03:37:08,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/4eda90d9af6c485b9c89ba8f5a2576c9, entries=250, sequenceid=425, filesize=16.8 K 2024-11-25T03:37:08,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/02ed19764cfe41ff8482396eaf764b9e as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/02ed19764cfe41ff8482396eaf764b9e 2024-11-25T03:37:08,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/02ed19764cfe41ff8482396eaf764b9e, entries=150, sequenceid=425, filesize=12.0 K 2024-11-25T03:37:08,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/acfe6dba1eb8466dbc8a29eb0f263458 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/acfe6dba1eb8466dbc8a29eb0f263458 2024-11-25T03:37:08,503 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/acfe6dba1eb8466dbc8a29eb0f263458, entries=150, sequenceid=425, filesize=12.0 K 2024-11-25T03:37:08,505 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for ee186277170c2bdfb070a7c209286857 in 523ms, sequenceid=425, compaction requested=true 2024-11-25T03:37:08,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:37:08,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee186277170c2bdfb070a7c209286857:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:37:08,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:08,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee186277170c2bdfb070a7c209286857:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:37:08,505 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:37:08,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:08,505 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:37:08,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee186277170c2bdfb070a7c209286857:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:37:08,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:37:08,510 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42703 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:37:08,510 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:37:08,510 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): ee186277170c2bdfb070a7c209286857/B is initiating minor compaction (all files) 2024-11-25T03:37:08,510 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): ee186277170c2bdfb070a7c209286857/A is initiating minor compaction (all files) 2024-11-25T03:37:08,510 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee186277170c2bdfb070a7c209286857/B in TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:08,510 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/8ff0a55df2ad4ff49ffb61591260b93d, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/6dc568a4ad3b46b0888283e5faad0b62, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/02ed19764cfe41ff8482396eaf764b9e] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp, totalSize=36.9 K 2024-11-25T03:37:08,511 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee186277170c2bdfb070a7c209286857/A in TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:08,511 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/ef7b5767eabd487ea1d47f718af9ef9b, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/3a466108ba57487a813214b8bb4ac583, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/4eda90d9af6c485b9c89ba8f5a2576c9] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp, totalSize=41.7 K 2024-11-25T03:37:08,511 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ff0a55df2ad4ff49ffb61591260b93d, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1732505825396 2024-11-25T03:37:08,511 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef7b5767eabd487ea1d47f718af9ef9b, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1732505825396 2024-11-25T03:37:08,511 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 6dc568a4ad3b46b0888283e5faad0b62, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1732505825604 2024-11-25T03:37:08,512 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a466108ba57487a813214b8bb4ac583, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1732505825604 2024-11-25T03:37:08,512 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 02ed19764cfe41ff8482396eaf764b9e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1732505826817 2024-11-25T03:37:08,512 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4eda90d9af6c485b9c89ba8f5a2576c9, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1732505826817 2024-11-25T03:37:08,524 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee186277170c2bdfb070a7c209286857#B#compaction#89 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:37:08,525 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/cf65fa346d074c3b9fe2cc96750a0d78 is 50, key is test_row_0/B:col10/1732505826830/Put/seqid=0 2024-11-25T03:37:08,539 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee186277170c2bdfb070a7c209286857#A#compaction#90 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:37:08,539 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/171def67df4047139f7eb7ecf821e3f7 is 50, key is test_row_0/A:col10/1732505826830/Put/seqid=0 2024-11-25T03:37:08,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741926_1102 (size=13323) 2024-11-25T03:37:08,563 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/cf65fa346d074c3b9fe2cc96750a0d78 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/cf65fa346d074c3b9fe2cc96750a0d78 2024-11-25T03:37:08,572 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee186277170c2bdfb070a7c209286857/B of ee186277170c2bdfb070a7c209286857 into cf65fa346d074c3b9fe2cc96750a0d78(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:37:08,572 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:37:08,572 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., storeName=ee186277170c2bdfb070a7c209286857/B, priority=13, startTime=1732505828505; duration=0sec 2024-11-25T03:37:08,572 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:37:08,572 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee186277170c2bdfb070a7c209286857:B 2024-11-25T03:37:08,572 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:37:08,574 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:37:08,574 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): ee186277170c2bdfb070a7c209286857/C is initiating minor compaction (all files) 2024-11-25T03:37:08,574 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee186277170c2bdfb070a7c209286857/C in TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:08,574 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/a6c3686fd0e64a5c8b54613daafef776, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/0081d65d2a1545a29fad9060968af23b, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/6b7133ab3f214cf9ba9c0caf9b24779d, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/acfe6dba1eb8466dbc8a29eb0f263458] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp, totalSize=48.9 K 2024-11-25T03:37:08,575 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting a6c3686fd0e64a5c8b54613daafef776, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1732505824738 2024-11-25T03:37:08,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741927_1103 (size=13323) 2024-11-25T03:37:08,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-25T03:37:08,576 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 0081d65d2a1545a29fad9060968af23b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1732505825396 2024-11-25T03:37:08,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:08,577 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing ee186277170c2bdfb070a7c209286857 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-25T03:37:08,577 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b7133ab3f214cf9ba9c0caf9b24779d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1732505825604 2024-11-25T03:37:08,580 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting acfe6dba1eb8466dbc8a29eb0f263458, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1732505826817 2024-11-25T03:37:08,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/2423ac6c69464abda0e01cdc5b80dc24 is 50, key is test_row_0/A:col10/1732505828000/Put/seqid=0 2024-11-25T03:37:08,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-25T03:37:08,592 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/171def67df4047139f7eb7ecf821e3f7 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/171def67df4047139f7eb7ecf821e3f7 2024-11-25T03:37:08,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741928_1104 (size=12301) 2024-11-25T03:37:08,602 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee186277170c2bdfb070a7c209286857/A of ee186277170c2bdfb070a7c209286857 into 171def67df4047139f7eb7ecf821e3f7(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:37:08,602 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:37:08,602 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., storeName=ee186277170c2bdfb070a7c209286857/A, priority=13, startTime=1732505828505; duration=0sec 2024-11-25T03:37:08,602 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:08,602 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee186277170c2bdfb070a7c209286857:A 2024-11-25T03:37:08,605 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee186277170c2bdfb070a7c209286857#C#compaction#92 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:37:08,606 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/2fcb049491dd435daf53b987e929e961 is 50, key is test_row_0/C:col10/1732505826830/Put/seqid=0 2024-11-25T03:37:08,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741929_1105 (size=13289) 2024-11-25T03:37:08,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on ee186277170c2bdfb070a7c209286857 2024-11-25T03:37:08,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. as already flushing 2024-11-25T03:37:08,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:08,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505888674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:08,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:08,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:08,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505888676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:08,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:08,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:08,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505888678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:08,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:08,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505888678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:08,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:08,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505888679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:08,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:08,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:08,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:08,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:08,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505888788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:08,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:08,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505888788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:08,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:08,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:08,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505888788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:08,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:08,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:08,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505888789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:08,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:08,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:08,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505888789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:08,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:08,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:08,995 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/2423ac6c69464abda0e01cdc5b80dc24 2024-11-25T03:37:08,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:08,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505888997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:08,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:08,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:08,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505888997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:08,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:08,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:08,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505888998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:08,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:08,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:09,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505888998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:09,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:09,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:09,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:09,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:09,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:09,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:09,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:09,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505888998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:09,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:09,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:09,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:09,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/7d522c1bf29e41f6a63aa0f7a4116a46 is 50, key is test_row_0/B:col10/1732505828000/Put/seqid=0 2024-11-25T03:37:09,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741930_1106 (size=12301) 2024-11-25T03:37:09,015 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/7d522c1bf29e41f6a63aa0f7a4116a46 2024-11-25T03:37:09,036 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/2fcb049491dd435daf53b987e929e961 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/2fcb049491dd435daf53b987e929e961 2024-11-25T03:37:09,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/55907ddbac2b421ca71e788a0cef9f9a is 50, key is test_row_0/C:col10/1732505828000/Put/seqid=0 2024-11-25T03:37:09,046 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee186277170c2bdfb070a7c209286857/C of ee186277170c2bdfb070a7c209286857 into 2fcb049491dd435daf53b987e929e961(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:37:09,046 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:37:09,046 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., storeName=ee186277170c2bdfb070a7c209286857/C, priority=12, startTime=1732505828505; duration=0sec 2024-11-25T03:37:09,047 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:09,047 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee186277170c2bdfb070a7c209286857:C 2024-11-25T03:37:09,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741931_1107 (size=12301) 2024-11-25T03:37:09,056 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/55907ddbac2b421ca71e788a0cef9f9a 2024-11-25T03:37:09,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/2423ac6c69464abda0e01cdc5b80dc24 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/2423ac6c69464abda0e01cdc5b80dc24 2024-11-25T03:37:09,072 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/2423ac6c69464abda0e01cdc5b80dc24, entries=150, sequenceid=440, filesize=12.0 K 2024-11-25T03:37:09,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/7d522c1bf29e41f6a63aa0f7a4116a46 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/7d522c1bf29e41f6a63aa0f7a4116a46 2024-11-25T03:37:09,084 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/7d522c1bf29e41f6a63aa0f7a4116a46, entries=150, sequenceid=440, filesize=12.0 K 2024-11-25T03:37:09,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/55907ddbac2b421ca71e788a0cef9f9a as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/55907ddbac2b421ca71e788a0cef9f9a 2024-11-25T03:37:09,095 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/55907ddbac2b421ca71e788a0cef9f9a, entries=150, sequenceid=440, filesize=12.0 K 2024-11-25T03:37:09,096 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for ee186277170c2bdfb070a7c209286857 in 520ms, sequenceid=440, compaction requested=false 2024-11-25T03:37:09,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:37:09,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:09,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-25T03:37:09,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-25T03:37:09,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-25T03:37:09,101 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-25T03:37:09,101 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1370 sec 2024-11-25T03:37:09,104 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.1440 sec 2024-11-25T03:37:09,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on ee186277170c2bdfb070a7c209286857 2024-11-25T03:37:09,311 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:37:09,311 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee186277170c2bdfb070a7c209286857 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-25T03:37:09,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/5cda8c5ceee848b5b283395f3792d68e is 50, key is test_row_0/A:col10/1732505829309/Put/seqid=0 2024-11-25T03:37:09,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:09,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505889320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:09,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:09,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505889324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:09,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:09,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:09,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:09,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:09,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:09,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505889328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:09,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:09,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:09,330 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:09,330 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:09,330 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:09,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:09,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505889329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:09,332 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:09,332 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:09,332 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:09,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:09,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505889331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:09,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:09,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:09,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:09,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741932_1108 (size=12301) 2024-11-25T03:37:09,337 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/5cda8c5ceee848b5b283395f3792d68e 2024-11-25T03:37:09,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/c39f42fb50274769a53c0ff6dfcc0f3c is 50, key is test_row_0/B:col10/1732505829309/Put/seqid=0 2024-11-25T03:37:09,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741933_1109 (size=12301) 2024-11-25T03:37:09,359 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/c39f42fb50274769a53c0ff6dfcc0f3c 2024-11-25T03:37:09,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/bd498a2839da4ba59451ec9b711c741d is 50, key is test_row_0/C:col10/1732505829309/Put/seqid=0 2024-11-25T03:37:09,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741934_1110 (size=12301) 2024-11-25T03:37:09,388 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/bd498a2839da4ba59451ec9b711c741d 2024-11-25T03:37:09,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/5cda8c5ceee848b5b283395f3792d68e as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/5cda8c5ceee848b5b283395f3792d68e 2024-11-25T03:37:09,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/5cda8c5ceee848b5b283395f3792d68e, entries=150, sequenceid=465, filesize=12.0 K 2024-11-25T03:37:09,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/c39f42fb50274769a53c0ff6dfcc0f3c as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/c39f42fb50274769a53c0ff6dfcc0f3c 2024-11-25T03:37:09,420 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/c39f42fb50274769a53c0ff6dfcc0f3c, entries=150, sequenceid=465, filesize=12.0 K 2024-11-25T03:37:09,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/bd498a2839da4ba59451ec9b711c741d as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/bd498a2839da4ba59451ec9b711c741d 2024-11-25T03:37:09,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/bd498a2839da4ba59451ec9b711c741d, entries=150, sequenceid=465, filesize=12.0 K 2024-11-25T03:37:09,431 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for ee186277170c2bdfb070a7c209286857 in 119ms, sequenceid=465, compaction requested=true 2024-11-25T03:37:09,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:37:09,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee186277170c2bdfb070a7c209286857:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:37:09,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:09,431 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:37:09,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee186277170c2bdfb070a7c209286857:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:37:09,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:09,431 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:37:09,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee186277170c2bdfb070a7c209286857:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:37:09,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:37:09,432 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:37:09,432 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:37:09,432 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): ee186277170c2bdfb070a7c209286857/B is initiating minor compaction (all files) 2024-11-25T03:37:09,432 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): ee186277170c2bdfb070a7c209286857/A is initiating minor compaction (all files) 2024-11-25T03:37:09,432 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee186277170c2bdfb070a7c209286857/B in TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:09,432 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee186277170c2bdfb070a7c209286857/A in TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:09,433 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/cf65fa346d074c3b9fe2cc96750a0d78, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/7d522c1bf29e41f6a63aa0f7a4116a46, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/c39f42fb50274769a53c0ff6dfcc0f3c] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp, totalSize=37.0 K 2024-11-25T03:37:09,433 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/171def67df4047139f7eb7ecf821e3f7, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/2423ac6c69464abda0e01cdc5b80dc24, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/5cda8c5ceee848b5b283395f3792d68e] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp, totalSize=37.0 K 2024-11-25T03:37:09,434 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting cf65fa346d074c3b9fe2cc96750a0d78, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1732505826817 2024-11-25T03:37:09,435 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 171def67df4047139f7eb7ecf821e3f7, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1732505826817 2024-11-25T03:37:09,435 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d522c1bf29e41f6a63aa0f7a4116a46, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1732505828000 2024-11-25T03:37:09,436 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2423ac6c69464abda0e01cdc5b80dc24, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1732505828000 2024-11-25T03:37:09,436 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting c39f42fb50274769a53c0ff6dfcc0f3c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1732505828676 2024-11-25T03:37:09,437 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cda8c5ceee848b5b283395f3792d68e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1732505828676 2024-11-25T03:37:09,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on ee186277170c2bdfb070a7c209286857 2024-11-25T03:37:09,443 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:37:09,443 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee186277170c2bdfb070a7c209286857 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-25T03:37:09,448 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee186277170c2bdfb070a7c209286857#B#compaction#98 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:37:09,449 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/33a864f7480244329da1bc4f5e9152ff is 50, key is test_row_0/B:col10/1732505829309/Put/seqid=0 2024-11-25T03:37:09,452 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee186277170c2bdfb070a7c209286857#A#compaction#99 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:37:09,452 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/94d7916d53164884b61051bd06b2133c is 50, key is test_row_0/A:col10/1732505829309/Put/seqid=0 2024-11-25T03:37:09,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/c2ab1f865f6d46bf8006b079c1734b73 is 50, key is test_row_0/A:col10/1732505829443/Put/seqid=0 2024-11-25T03:37:09,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:09,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505889470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:09,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:09,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505889471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:09,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:09,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:09,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:09,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:09,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505889472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:09,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:09,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:09,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:09,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:09,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:09,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:09,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:09,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505889473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:09,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:09,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505889475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:09,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:09,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:09,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:09,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:09,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:09,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:09,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741935_1111 (size=13425) 2024-11-25T03:37:09,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741936_1112 (size=12301) 2024-11-25T03:37:09,493 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/c2ab1f865f6d46bf8006b079c1734b73 2024-11-25T03:37:09,498 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/33a864f7480244329da1bc4f5e9152ff as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/33a864f7480244329da1bc4f5e9152ff 2024-11-25T03:37:09,508 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee186277170c2bdfb070a7c209286857/B of ee186277170c2bdfb070a7c209286857 into 33a864f7480244329da1bc4f5e9152ff(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:37:09,508 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:37:09,508 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., storeName=ee186277170c2bdfb070a7c209286857/B, priority=13, startTime=1732505829431; duration=0sec 2024-11-25T03:37:09,508 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:37:09,508 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee186277170c2bdfb070a7c209286857:B 2024-11-25T03:37:09,508 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:37:09,510 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:37:09,510 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): ee186277170c2bdfb070a7c209286857/C is initiating minor compaction (all files) 2024-11-25T03:37:09,510 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee186277170c2bdfb070a7c209286857/C in TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:09,510 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/2fcb049491dd435daf53b987e929e961, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/55907ddbac2b421ca71e788a0cef9f9a, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/bd498a2839da4ba59451ec9b711c741d] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp, totalSize=37.0 K 2024-11-25T03:37:09,511 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fcb049491dd435daf53b987e929e961, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1732505826817 2024-11-25T03:37:09,512 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 55907ddbac2b421ca71e788a0cef9f9a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1732505828000 2024-11-25T03:37:09,512 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting bd498a2839da4ba59451ec9b711c741d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1732505828676 2024-11-25T03:37:09,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741937_1113 (size=13425) 2024-11-25T03:37:09,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/102ba5317af94e13b0280e1f5f827e93 is 50, key is test_row_0/B:col10/1732505829443/Put/seqid=0 2024-11-25T03:37:09,523 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/94d7916d53164884b61051bd06b2133c as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/94d7916d53164884b61051bd06b2133c 2024-11-25T03:37:09,538 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee186277170c2bdfb070a7c209286857#C#compaction#102 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:37:09,538 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee186277170c2bdfb070a7c209286857/A of ee186277170c2bdfb070a7c209286857 into 94d7916d53164884b61051bd06b2133c(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:37:09,539 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/50a03085ddf345979d708a7008fef6cf is 50, key is test_row_0/C:col10/1732505829309/Put/seqid=0 2024-11-25T03:37:09,540 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:37:09,540 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., storeName=ee186277170c2bdfb070a7c209286857/A, priority=13, startTime=1732505829431; duration=0sec 2024-11-25T03:37:09,540 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:09,540 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee186277170c2bdfb070a7c209286857:A 2024-11-25T03:37:09,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741938_1114 (size=12301) 2024-11-25T03:37:09,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741939_1115 (size=13391) 2024-11-25T03:37:09,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:09,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505889587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:09,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:09,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505889588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:09,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:09,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:09,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:09,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:09,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:09,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505889588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:09,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:09,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:09,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:09,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:09,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:09,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505889588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:09,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:09,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:09,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:09,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:09,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:09,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505889588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:09,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:09,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:09,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:09,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:09,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505889797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:09,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:09,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:09,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:09,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505889798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:09,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:09,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:09,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:09,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:09,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:09,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505889798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:09,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:09,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:09,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:09,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:09,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505889798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:09,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:09,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505889800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:09,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:09,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:09,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:09,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:09,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:09,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:09,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/102ba5317af94e13b0280e1f5f827e93 2024-11-25T03:37:09,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/04b46cd6845e4de99bab3557d5dab8bf is 50, key is test_row_0/C:col10/1732505829443/Put/seqid=0 2024-11-25T03:37:09,964 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/50a03085ddf345979d708a7008fef6cf as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/50a03085ddf345979d708a7008fef6cf 2024-11-25T03:37:09,970 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee186277170c2bdfb070a7c209286857/C of ee186277170c2bdfb070a7c209286857 into 50a03085ddf345979d708a7008fef6cf(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:37:09,970 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:37:09,971 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., storeName=ee186277170c2bdfb070a7c209286857/C, priority=13, startTime=1732505829431; duration=0sec 2024-11-25T03:37:09,971 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:09,971 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee186277170c2bdfb070a7c209286857:C 2024-11-25T03:37:09,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741940_1116 (size=12301) 2024-11-25T03:37:10,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:10,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505890108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:10,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:10,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505890108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:10,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:10,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:10,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:10,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:10,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505890108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:10,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:10,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:10,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:10,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505890108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:10,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:10,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:10,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505890108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:10,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:10,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:10,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:10,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:10,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:10,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:10,111 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:10,111 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:10,111 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:10,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-25T03:37:10,118 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:37:10,118 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:37:10,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-11-25T03:37:10,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-25T03:37:10,122 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:37:10,122 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:37:10,122 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:37:10,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-25T03:37:10,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-25T03:37:10,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:10,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. as already flushing 2024-11-25T03:37:10,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:10,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:10,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:10,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:10,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/04b46cd6845e4de99bab3557d5dab8bf 2024-11-25T03:37:10,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/c2ab1f865f6d46bf8006b079c1734b73 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/c2ab1f865f6d46bf8006b079c1734b73 2024-11-25T03:37:10,390 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/c2ab1f865f6d46bf8006b079c1734b73, entries=150, sequenceid=481, filesize=12.0 K 2024-11-25T03:37:10,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/102ba5317af94e13b0280e1f5f827e93 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/102ba5317af94e13b0280e1f5f827e93 2024-11-25T03:37:10,399 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/102ba5317af94e13b0280e1f5f827e93, entries=150, sequenceid=481, filesize=12.0 K 2024-11-25T03:37:10,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/04b46cd6845e4de99bab3557d5dab8bf as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/04b46cd6845e4de99bab3557d5dab8bf 2024-11-25T03:37:10,408 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/04b46cd6845e4de99bab3557d5dab8bf, entries=150, sequenceid=481, filesize=12.0 K 2024-11-25T03:37:10,409 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for ee186277170c2bdfb070a7c209286857 in 966ms, sequenceid=481, compaction requested=false 2024-11-25T03:37:10,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:37:10,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-25T03:37:10,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:10,431 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing ee186277170c2bdfb070a7c209286857 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-25T03:37:10,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-25T03:37:10,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/703b445cc5e84eb1aee104369f0e9c16 is 50, key is test_row_0/A:col10/1732505829471/Put/seqid=0 2024-11-25T03:37:10,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741941_1117 (size=12301) 2024-11-25T03:37:10,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on ee186277170c2bdfb070a7c209286857 2024-11-25T03:37:10,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. as already flushing 2024-11-25T03:37:10,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:10,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505890627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:10,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:10,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:10,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:10,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:10,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505890628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:10,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:10,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:10,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:10,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:10,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505890647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:10,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:10,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:10,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505890649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:10,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:10,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:10,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:10,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:10,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:10,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:10,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505890651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:10,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:10,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:10,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:10,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-25T03:37:10,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:10,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505890757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:10,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:10,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:10,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:10,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505890757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:10,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:10,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:10,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:10,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:10,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:10,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505890757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:10,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:10,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:10,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:10,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:10,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505890758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:10,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:10,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505890758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:10,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:10,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:10,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:10,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:10,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:10,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:10,862 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=504 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/703b445cc5e84eb1aee104369f0e9c16 2024-11-25T03:37:10,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/b385b814e5ae4aeb8df60d18e5eca324 is 50, key is test_row_0/B:col10/1732505829471/Put/seqid=0 2024-11-25T03:37:10,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741942_1118 (size=12301) 2024-11-25T03:37:10,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:10,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505890967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:10,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:10,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505890967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:10,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:10,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:10,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:10,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:10,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:10,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:10,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505890968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:10,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:10,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:10,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:10,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:10,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:10,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505890968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:10,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:10,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:10,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:10,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:10,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505890979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:10,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:10,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:10,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:11,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-25T03:37:11,278 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=504 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/b385b814e5ae4aeb8df60d18e5eca324 2024-11-25T03:37:11,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:11,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505891278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:11,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:11,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:11,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:11,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505891278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:11,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:11,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:11,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505891278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:11,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:11,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:11,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:11,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:11,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:11,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:11,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:11,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505891287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:11,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:11,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:11,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:11,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:11,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505891287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:11,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:11,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:11,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:11,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/b913421b75184dcf9663ddd968b832eb is 50, key is test_row_0/C:col10/1732505829471/Put/seqid=0 2024-11-25T03:37:11,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741943_1119 (size=12301) 2024-11-25T03:37:11,708 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=504 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/b913421b75184dcf9663ddd968b832eb 2024-11-25T03:37:11,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/703b445cc5e84eb1aee104369f0e9c16 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/703b445cc5e84eb1aee104369f0e9c16 2024-11-25T03:37:11,720 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/703b445cc5e84eb1aee104369f0e9c16, entries=150, sequenceid=504, filesize=12.0 K 2024-11-25T03:37:11,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/b385b814e5ae4aeb8df60d18e5eca324 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/b385b814e5ae4aeb8df60d18e5eca324 2024-11-25T03:37:11,728 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/b385b814e5ae4aeb8df60d18e5eca324, entries=150, sequenceid=504, filesize=12.0 K 2024-11-25T03:37:11,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/b913421b75184dcf9663ddd968b832eb as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/b913421b75184dcf9663ddd968b832eb 2024-11-25T03:37:11,736 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/b913421b75184dcf9663ddd968b832eb, entries=150, sequenceid=504, filesize=12.0 K 2024-11-25T03:37:11,737 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for ee186277170c2bdfb070a7c209286857 in 1306ms, sequenceid=504, compaction requested=true 2024-11-25T03:37:11,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:37:11,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:11,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-11-25T03:37:11,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-11-25T03:37:11,741 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-25T03:37:11,741 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6160 sec 2024-11-25T03:37:11,744 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 1.6240 sec 2024-11-25T03:37:11,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on ee186277170c2bdfb070a7c209286857 2024-11-25T03:37:11,792 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:37:11,793 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee186277170c2bdfb070a7c209286857 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-25T03:37:11,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/fbfed05dfb194d61868bb657295cf4f6 is 50, key is test_row_0/A:col10/1732505830648/Put/seqid=0 2024-11-25T03:37:11,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:11,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505891814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:11,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:11,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:11,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:11,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:11,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505891821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:11,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:11,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:11,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:11,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:11,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505891823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:11,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:11,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505891826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:11,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:11,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:11,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:11,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:11,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505891826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:11,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:11,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:11,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:11,836 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:11,836 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:11,836 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:11,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741944_1120 (size=14741) 2024-11-25T03:37:11,848 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/fbfed05dfb194d61868bb657295cf4f6 2024-11-25T03:37:11,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/e2ad2634834744aebe6e1f3ea6ec9179 is 50, key is test_row_0/B:col10/1732505830648/Put/seqid=0 2024-11-25T03:37:11,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741945_1121 (size=12301) 2024-11-25T03:37:11,869 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/e2ad2634834744aebe6e1f3ea6ec9179 2024-11-25T03:37:11,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/2c2103d741874c6e825a56c4b0f4f4c9 is 50, key is test_row_0/C:col10/1732505830648/Put/seqid=0 2024-11-25T03:37:11,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741946_1122 (size=12301) 2024-11-25T03:37:11,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:11,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505891937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:11,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:11,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:11,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505891937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:11,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:11,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:11,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:11,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:11,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:11,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:11,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505891947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:11,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:11,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:11,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:11,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505891947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:11,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:11,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:11,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:11,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:11,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:11,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505891947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:11,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:11,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:11,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:12,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:12,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505892148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:12,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:12,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:12,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:12,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:12,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505892148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:12,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:12,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:12,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:12,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:12,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505892158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:12,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:12,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:12,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:12,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:12,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505892158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:12,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:12,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:12,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:12,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:12,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505892158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:12,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:12,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:12,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:12,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-25T03:37:12,278 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:37:12,278 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:37:12,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees 2024-11-25T03:37:12,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-25T03:37:12,281 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=29, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:37:12,284 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=29, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:37:12,284 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:37:12,306 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/2c2103d741874c6e825a56c4b0f4f4c9 2024-11-25T03:37:12,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/fbfed05dfb194d61868bb657295cf4f6 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/fbfed05dfb194d61868bb657295cf4f6 2024-11-25T03:37:12,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/fbfed05dfb194d61868bb657295cf4f6, entries=200, sequenceid=523, filesize=14.4 K 2024-11-25T03:37:12,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/e2ad2634834744aebe6e1f3ea6ec9179 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/e2ad2634834744aebe6e1f3ea6ec9179 2024-11-25T03:37:12,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/e2ad2634834744aebe6e1f3ea6ec9179, entries=150, sequenceid=523, filesize=12.0 K 2024-11-25T03:37:12,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/2c2103d741874c6e825a56c4b0f4f4c9 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/2c2103d741874c6e825a56c4b0f4f4c9 2024-11-25T03:37:12,351 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/2c2103d741874c6e825a56c4b0f4f4c9, entries=150, sequenceid=523, filesize=12.0 K 2024-11-25T03:37:12,364 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for ee186277170c2bdfb070a7c209286857 in 571ms, sequenceid=523, compaction requested=true 2024-11-25T03:37:12,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:37:12,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee186277170c2bdfb070a7c209286857:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:37:12,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:12,364 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:37:12,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee186277170c2bdfb070a7c209286857:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:37:12,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:12,364 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:37:12,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee186277170c2bdfb070a7c209286857:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:37:12,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:37:12,366 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52768 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:37:12,366 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): ee186277170c2bdfb070a7c209286857/A is initiating minor compaction (all files) 2024-11-25T03:37:12,366 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee186277170c2bdfb070a7c209286857/A in TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:12,367 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/94d7916d53164884b61051bd06b2133c, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/c2ab1f865f6d46bf8006b079c1734b73, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/703b445cc5e84eb1aee104369f0e9c16, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/fbfed05dfb194d61868bb657295cf4f6] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp, totalSize=51.5 K 2024-11-25T03:37:12,367 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50328 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:37:12,367 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): ee186277170c2bdfb070a7c209286857/B is initiating minor compaction (all files) 2024-11-25T03:37:12,368 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee186277170c2bdfb070a7c209286857/B in TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:12,368 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/33a864f7480244329da1bc4f5e9152ff, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/102ba5317af94e13b0280e1f5f827e93, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/b385b814e5ae4aeb8df60d18e5eca324, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/e2ad2634834744aebe6e1f3ea6ec9179] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp, totalSize=49.1 K 2024-11-25T03:37:12,368 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94d7916d53164884b61051bd06b2133c, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1732505828676 2024-11-25T03:37:12,368 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 33a864f7480244329da1bc4f5e9152ff, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1732505828676 2024-11-25T03:37:12,368 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2ab1f865f6d46bf8006b079c1734b73, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1732505829326 2024-11-25T03:37:12,369 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 102ba5317af94e13b0280e1f5f827e93, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1732505829326 2024-11-25T03:37:12,369 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 703b445cc5e84eb1aee104369f0e9c16, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=504, earliestPutTs=1732505829469 2024-11-25T03:37:12,370 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbfed05dfb194d61868bb657295cf4f6, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1732505830648 2024-11-25T03:37:12,370 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting b385b814e5ae4aeb8df60d18e5eca324, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=504, earliestPutTs=1732505829469 2024-11-25T03:37:12,371 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting e2ad2634834744aebe6e1f3ea6ec9179, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1732505830648 2024-11-25T03:37:12,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-25T03:37:12,395 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee186277170c2bdfb070a7c209286857#B#compaction#110 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:37:12,395 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/9859ed15e92547adbe0e7a2427cba252 is 50, key is test_row_0/B:col10/1732505830648/Put/seqid=0 2024-11-25T03:37:12,403 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee186277170c2bdfb070a7c209286857#A#compaction#111 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:37:12,404 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/36dec1ebe17a470cb74a18110105119f is 50, key is test_row_0/A:col10/1732505830648/Put/seqid=0 2024-11-25T03:37:12,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741947_1123 (size=13561) 2024-11-25T03:37:12,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741948_1124 (size=13561) 2024-11-25T03:37:12,430 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/9859ed15e92547adbe0e7a2427cba252 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/9859ed15e92547adbe0e7a2427cba252 2024-11-25T03:37:12,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-25T03:37:12,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:12,439 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2902): Flushing ee186277170c2bdfb070a7c209286857 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-25T03:37:12,440 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/36dec1ebe17a470cb74a18110105119f as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/36dec1ebe17a470cb74a18110105119f 2024-11-25T03:37:12,452 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee186277170c2bdfb070a7c209286857/B of ee186277170c2bdfb070a7c209286857 into 9859ed15e92547adbe0e7a2427cba252(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:37:12,452 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee186277170c2bdfb070a7c209286857/A of ee186277170c2bdfb070a7c209286857 into 36dec1ebe17a470cb74a18110105119f(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:37:12,452 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:37:12,452 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:37:12,452 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., storeName=ee186277170c2bdfb070a7c209286857/B, priority=12, startTime=1732505832364; duration=0sec 2024-11-25T03:37:12,452 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., storeName=ee186277170c2bdfb070a7c209286857/A, priority=12, startTime=1732505832364; duration=0sec 2024-11-25T03:37:12,452 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:37:12,452 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee186277170c2bdfb070a7c209286857:B 2024-11-25T03:37:12,452 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:37:12,452 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee186277170c2bdfb070a7c209286857:A 2024-11-25T03:37:12,452 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:37:12,454 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50294 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:37:12,454 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): ee186277170c2bdfb070a7c209286857/C is initiating minor compaction (all files) 2024-11-25T03:37:12,454 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee186277170c2bdfb070a7c209286857/C in TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:12,455 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/50a03085ddf345979d708a7008fef6cf, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/04b46cd6845e4de99bab3557d5dab8bf, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/b913421b75184dcf9663ddd968b832eb, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/2c2103d741874c6e825a56c4b0f4f4c9] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp, totalSize=49.1 K 2024-11-25T03:37:12,457 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 50a03085ddf345979d708a7008fef6cf, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1732505828676 2024-11-25T03:37:12,458 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 04b46cd6845e4de99bab3557d5dab8bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1732505829326 2024-11-25T03:37:12,458 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting b913421b75184dcf9663ddd968b832eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=504, earliestPutTs=1732505829469 2024-11-25T03:37:12,459 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c2103d741874c6e825a56c4b0f4f4c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1732505830648 2024-11-25T03:37:12,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/76e52d1d40d14ca494c0b96f94b912df is 50, key is test_row_0/A:col10/1732505831823/Put/seqid=0 2024-11-25T03:37:12,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. as already flushing 2024-11-25T03:37:12,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on ee186277170c2bdfb070a7c209286857 2024-11-25T03:37:12,482 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee186277170c2bdfb070a7c209286857#C#compaction#113 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:37:12,483 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/2930a4bd6f5d405ca761b61cdf610e37 is 50, key is test_row_0/C:col10/1732505830648/Put/seqid=0 2024-11-25T03:37:12,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:12,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60596 deadline: 1732505892493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:12,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:12,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:12,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60588 deadline: 1732505892495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:12,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:12,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:12,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:12,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:12,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:12,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:12,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60572 deadline: 1732505892497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:12,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:12,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:12,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:12,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741949_1125 (size=12301) 2024-11-25T03:37:12,505 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/76e52d1d40d14ca494c0b96f94b912df 2024-11-25T03:37:12,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:12,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60560 deadline: 1732505892499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:12,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:12,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:12,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:12,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:12,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60576 deadline: 1732505892501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:12,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:12,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:12,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:12,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/b767147431ca4008bceba98b49cd95cf is 50, key is test_row_0/B:col10/1732505831823/Put/seqid=0 2024-11-25T03:37:12,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741950_1126 (size=13527) 2024-11-25T03:37:12,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741951_1127 (size=12301) 2024-11-25T03:37:12,544 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/b767147431ca4008bceba98b49cd95cf 2024-11-25T03:37:12,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/d3c793b0ad76436cbd157099b76bb484 is 50, key is test_row_0/C:col10/1732505831823/Put/seqid=0 2024-11-25T03:37:12,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741952_1128 (size=12301) 2024-11-25T03:37:12,562 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/d3c793b0ad76436cbd157099b76bb484 2024-11-25T03:37:12,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/76e52d1d40d14ca494c0b96f94b912df as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/76e52d1d40d14ca494c0b96f94b912df 2024-11-25T03:37:12,576 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/76e52d1d40d14ca494c0b96f94b912df, entries=150, sequenceid=540, filesize=12.0 K 2024-11-25T03:37:12,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/b767147431ca4008bceba98b49cd95cf as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/b767147431ca4008bceba98b49cd95cf 2024-11-25T03:37:12,584 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/b767147431ca4008bceba98b49cd95cf, entries=150, sequenceid=540, filesize=12.0 K 2024-11-25T03:37:12,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/d3c793b0ad76436cbd157099b76bb484 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/d3c793b0ad76436cbd157099b76bb484 2024-11-25T03:37:12,593 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/d3c793b0ad76436cbd157099b76bb484, entries=150, sequenceid=540, filesize=12.0 K 2024-11-25T03:37:12,594 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for ee186277170c2bdfb070a7c209286857 in 155ms, sequenceid=540, compaction requested=false 2024-11-25T03:37:12,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2603): Flush status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:37:12,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:12,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=30 2024-11-25T03:37:12,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4169): Remote procedure done, pid=30 2024-11-25T03:37:12,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-25T03:37:12,598 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-25T03:37:12,598 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 311 msec 2024-11-25T03:37:12,600 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees in 321 msec 2024-11-25T03:37:12,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on ee186277170c2bdfb070a7c209286857 2024-11-25T03:37:12,611 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:37:12,611 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee186277170c2bdfb070a7c209286857 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-25T03:37:12,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/4b06b2d2030b40a3b9e1f3fa5562ad46 is 50, key is test_row_0/A:col10/1732505832499/Put/seqid=0 2024-11-25T03:37:12,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741953_1129 (size=14741) 2024-11-25T03:37:12,651 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-25T03:37:12,651 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:37:12,655 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-25T03:37:12,656 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:37:12,656 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-25T03:37:12,656 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:37:12,657 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-25T03:37:12,657 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:37:12,658 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-25T03:37:12,658 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:37:12,658 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:12,658 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:12,658 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:12,658 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:12,659 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:12,659 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:12,659 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:12,659 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:37:12,659 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:37:12,659 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:37:12,659 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:12,659 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-25T03:37:12,659 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:37:12,659 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:37:12,659 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:12,659 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:12,659 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:12,659 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:12,659 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:37:12,660 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:37:12,661 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-25T03:37:12,661 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:37:12,661 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:12,661 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-25T03:37:12,661 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:37:12,661 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:12,661 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:12,661 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:12,661 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:37:12,662 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-25T03:37:12,662 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:37:12,662 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:12,662 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:12,662 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:37:12,662 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:37:12,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-25T03:37:12,908 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:37:12,908 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-25T03:37:12,908 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 94 2024-11-25T03:37:12,908 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 83 2024-11-25T03:37:12,908 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 96 2024-11-25T03:37:12,908 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-11-25T03:37:12,908 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-25T03:37:12,908 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-25T03:37:12,908 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2823 2024-11-25T03:37:12,908 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2777 2024-11-25T03:37:12,909 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-25T03:37:12,909 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1291 2024-11-25T03:37:12,909 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3863 rows 2024-11-25T03:37:12,909 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1296 2024-11-25T03:37:12,909 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3869 rows 2024-11-25T03:37:12,909 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T03:37:12,909 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T03:37:12,909 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:12,909 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:12,909 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:37:12,914 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-25T03:37:12,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-25T03:37:12,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-25T03:37:12,922 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732505832922"}]},"ts":"1732505832922"} 2024-11-25T03:37:12,924 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-25T03:37:12,924 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-25T03:37:12,926 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T03:37:12,931 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee186277170c2bdfb070a7c209286857, UNASSIGN}] 2024-11-25T03:37:12,932 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee186277170c2bdfb070a7c209286857, UNASSIGN 2024-11-25T03:37:12,933 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=ee186277170c2bdfb070a7c209286857, regionState=CLOSING, regionLocation=38a5d2077268,34881,1732505806588 2024-11-25T03:37:12,935 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee186277170c2bdfb070a7c209286857, UNASSIGN because future has completed 2024-11-25T03:37:12,936 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T03:37:12,936 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588}] 2024-11-25T03:37:12,941 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/2930a4bd6f5d405ca761b61cdf610e37 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/2930a4bd6f5d405ca761b61cdf610e37 2024-11-25T03:37:12,947 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee186277170c2bdfb070a7c209286857/C of ee186277170c2bdfb070a7c209286857 into 2930a4bd6f5d405ca761b61cdf610e37(size=13.2 K), total size for store is 25.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:37:12,947 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:37:12,947 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., storeName=ee186277170c2bdfb070a7c209286857/C, priority=12, startTime=1732505832364; duration=0sec 2024-11-25T03:37:12,947 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:12,947 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee186277170c2bdfb070a7c209286857:C 2024-11-25T03:37:13,023 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=562 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/4b06b2d2030b40a3b9e1f3fa5562ad46 2024-11-25T03:37:13,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-25T03:37:13,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/2f72585f4138417794e2e427432544d6 is 50, key is test_row_0/B:col10/1732505832499/Put/seqid=0 2024-11-25T03:37:13,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741954_1130 (size=12301) 2024-11-25T03:37:13,093 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(122): Close ee186277170c2bdfb070a7c209286857 2024-11-25T03:37:13,094 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T03:37:13,094 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1722): Closing ee186277170c2bdfb070a7c209286857, disabling compactions & flushes 2024-11-25T03:37:13,094 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:13,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-25T03:37:13,437 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=562 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/2f72585f4138417794e2e427432544d6 2024-11-25T03:37:13,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/a9cf3ea10111474d97bbe34abedfdb68 is 50, key is test_row_0/C:col10/1732505832499/Put/seqid=0 2024-11-25T03:37:13,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741955_1131 (size=12301) 2024-11-25T03:37:13,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-25T03:37:13,851 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=562 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/a9cf3ea10111474d97bbe34abedfdb68 2024-11-25T03:37:13,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/4b06b2d2030b40a3b9e1f3fa5562ad46 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/4b06b2d2030b40a3b9e1f3fa5562ad46 2024-11-25T03:37:13,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/4b06b2d2030b40a3b9e1f3fa5562ad46, entries=200, sequenceid=562, filesize=14.4 K 2024-11-25T03:37:13,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/2f72585f4138417794e2e427432544d6 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/2f72585f4138417794e2e427432544d6 2024-11-25T03:37:13,866 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/2f72585f4138417794e2e427432544d6, entries=150, sequenceid=562, filesize=12.0 K 2024-11-25T03:37:13,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/a9cf3ea10111474d97bbe34abedfdb68 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/a9cf3ea10111474d97bbe34abedfdb68 2024-11-25T03:37:13,870 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/a9cf3ea10111474d97bbe34abedfdb68, entries=150, sequenceid=562, filesize=12.0 K 2024-11-25T03:37:13,871 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for ee186277170c2bdfb070a7c209286857 in 1260ms, sequenceid=562, compaction requested=true 2024-11-25T03:37:13,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee186277170c2bdfb070a7c209286857: 2024-11-25T03:37:13,871 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:13,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee186277170c2bdfb070a7c209286857:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:37:13,872 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:13,872 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. because compaction request was cancelled 2024-11-25T03:37:13,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:13,872 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee186277170c2bdfb070a7c209286857:A 2024-11-25T03:37:13,872 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. because compaction request was cancelled 2024-11-25T03:37:13,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee186277170c2bdfb070a7c209286857:B, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:37:13,872 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. after waiting 0 ms 2024-11-25T03:37:13,872 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee186277170c2bdfb070a7c209286857:B 2024-11-25T03:37:13,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:13,872 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:13,872 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. because compaction request was cancelled 2024-11-25T03:37:13,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee186277170c2bdfb070a7c209286857:C, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:37:13,872 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee186277170c2bdfb070a7c209286857:C 2024-11-25T03:37:13,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:13,872 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(2902): Flushing ee186277170c2bdfb070a7c209286857 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-25T03:37:13,876 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/4541766953a64798b38dbf0bd4dd73c2 is 50, key is test_row_0/A:col10/1732505832658/Put/seqid=0 2024-11-25T03:37:13,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741956_1132 (size=12301) 2024-11-25T03:37:14,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-25T03:37:14,280 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=578 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/4541766953a64798b38dbf0bd4dd73c2 2024-11-25T03:37:14,288 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/fd411f47f801470584324e3ae609fd49 is 50, key is test_row_0/B:col10/1732505832658/Put/seqid=0 2024-11-25T03:37:14,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741957_1133 (size=12301) 2024-11-25T03:37:14,692 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=578 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/fd411f47f801470584324e3ae609fd49 2024-11-25T03:37:14,699 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/4a02fa5288de48b0a4f7ed0f6c4baddc is 50, key is test_row_0/C:col10/1732505832658/Put/seqid=0 2024-11-25T03:37:14,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741958_1134 (size=12301) 2024-11-25T03:37:14,798 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T03:37:15,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-25T03:37:15,104 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=578 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/4a02fa5288de48b0a4f7ed0f6c4baddc 2024-11-25T03:37:15,109 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/A/4541766953a64798b38dbf0bd4dd73c2 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/4541766953a64798b38dbf0bd4dd73c2 2024-11-25T03:37:15,114 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/4541766953a64798b38dbf0bd4dd73c2, entries=150, sequenceid=578, filesize=12.0 K 2024-11-25T03:37:15,115 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/B/fd411f47f801470584324e3ae609fd49 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/fd411f47f801470584324e3ae609fd49 2024-11-25T03:37:15,120 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/fd411f47f801470584324e3ae609fd49, entries=150, sequenceid=578, filesize=12.0 K 2024-11-25T03:37:15,121 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/.tmp/C/4a02fa5288de48b0a4f7ed0f6c4baddc as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/4a02fa5288de48b0a4f7ed0f6c4baddc 2024-11-25T03:37:15,125 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/4a02fa5288de48b0a4f7ed0f6c4baddc, entries=150, sequenceid=578, filesize=12.0 K 2024-11-25T03:37:15,126 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=0 B/0 for ee186277170c2bdfb070a7c209286857 in 1254ms, sequenceid=578, compaction requested=true 2024-11-25T03:37:15,127 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/7298417c4ab745ddb2f32b0c8a12c586, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/9cc336ed6808431d803a3116580beb05, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/ab0a6e86e32e4b8db351200d8bd42d47, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/4ccec1bd1585433aa572d844cca23ddb, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/f46bc099f04745e3930cd831c7a8791d, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/8ff3acbe06de44d4910f0f40bac50e4b, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/2e4dd8ef19cf44a7a4767df5d4225ddb, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/f85b014f3c6d467fa0ac2c7b9fecca1f, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/f2e1ce89e5b24173bd82727e71dd097e, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/d75be72d1b33439a8bccce9798519d89, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/8962bbe8a13246aa81b5bdccc983867c, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/b5202a1881134fcf83d17717ffeae3db, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/8738b017b0bc467484c9a06327421192, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/90ddb124ef7a48f683ba7cde65a047a2, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/343b88cc95db4a36b10c3ae5f0b80fc0, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/a6f338444f0f459ca3403c7a501cc9a2, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/88130f4e637e4847ac02fbab4290b511, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/bedbe9c9b12341258acab2238fbfbb90, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/eadb6bdb99e947d1aa7101218f025793, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/1d4a111bf0b045d1ac34cf2d14072ea5, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/8a2954eabeb94f54a9371b30bc7244f9, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/55ea99d3fef445ba9751d5b90eb32cfc, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/b7bd8e46c82e44e4889492c77808ec26, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/f4eb4f81c84e4fc58b7b5afbc270aa99, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/5fc6ff77e2ad4dc2bd5af8c06c608734, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/75389c1961f844dfad5c8bb8d5f7c9c0, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/598260aa127c4d208a30e10c193a924b, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/ef7b5767eabd487ea1d47f718af9ef9b, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/3a466108ba57487a813214b8bb4ac583, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/4eda90d9af6c485b9c89ba8f5a2576c9, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/171def67df4047139f7eb7ecf821e3f7, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/2423ac6c69464abda0e01cdc5b80dc24, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/94d7916d53164884b61051bd06b2133c, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/5cda8c5ceee848b5b283395f3792d68e, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/c2ab1f865f6d46bf8006b079c1734b73, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/703b445cc5e84eb1aee104369f0e9c16, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/fbfed05dfb194d61868bb657295cf4f6] to archive 2024-11-25T03:37:15,130 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T03:37:15,137 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/7298417c4ab745ddb2f32b0c8a12c586 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/7298417c4ab745ddb2f32b0c8a12c586 2024-11-25T03:37:15,138 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/9cc336ed6808431d803a3116580beb05 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/9cc336ed6808431d803a3116580beb05 2024-11-25T03:37:15,139 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/ab0a6e86e32e4b8db351200d8bd42d47 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/ab0a6e86e32e4b8db351200d8bd42d47 2024-11-25T03:37:15,141 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/4ccec1bd1585433aa572d844cca23ddb to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/4ccec1bd1585433aa572d844cca23ddb 2024-11-25T03:37:15,142 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/f46bc099f04745e3930cd831c7a8791d to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/f46bc099f04745e3930cd831c7a8791d 2024-11-25T03:37:15,143 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/8ff3acbe06de44d4910f0f40bac50e4b to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/8ff3acbe06de44d4910f0f40bac50e4b 2024-11-25T03:37:15,145 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/2e4dd8ef19cf44a7a4767df5d4225ddb to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/2e4dd8ef19cf44a7a4767df5d4225ddb 2024-11-25T03:37:15,146 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/f85b014f3c6d467fa0ac2c7b9fecca1f to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/f85b014f3c6d467fa0ac2c7b9fecca1f 2024-11-25T03:37:15,147 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/f2e1ce89e5b24173bd82727e71dd097e to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/f2e1ce89e5b24173bd82727e71dd097e 2024-11-25T03:37:15,148 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/d75be72d1b33439a8bccce9798519d89 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/d75be72d1b33439a8bccce9798519d89 2024-11-25T03:37:15,149 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/8962bbe8a13246aa81b5bdccc983867c to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/8962bbe8a13246aa81b5bdccc983867c 2024-11-25T03:37:15,150 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/b5202a1881134fcf83d17717ffeae3db to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/b5202a1881134fcf83d17717ffeae3db 2024-11-25T03:37:15,151 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/8738b017b0bc467484c9a06327421192 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/8738b017b0bc467484c9a06327421192 2024-11-25T03:37:15,153 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/90ddb124ef7a48f683ba7cde65a047a2 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/90ddb124ef7a48f683ba7cde65a047a2 2024-11-25T03:37:15,154 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/343b88cc95db4a36b10c3ae5f0b80fc0 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/343b88cc95db4a36b10c3ae5f0b80fc0 2024-11-25T03:37:15,155 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/a6f338444f0f459ca3403c7a501cc9a2 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/a6f338444f0f459ca3403c7a501cc9a2 2024-11-25T03:37:15,157 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/88130f4e637e4847ac02fbab4290b511 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/88130f4e637e4847ac02fbab4290b511 2024-11-25T03:37:15,158 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/bedbe9c9b12341258acab2238fbfbb90 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/bedbe9c9b12341258acab2238fbfbb90 2024-11-25T03:37:15,159 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/eadb6bdb99e947d1aa7101218f025793 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/eadb6bdb99e947d1aa7101218f025793 2024-11-25T03:37:15,160 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/1d4a111bf0b045d1ac34cf2d14072ea5 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/1d4a111bf0b045d1ac34cf2d14072ea5 2024-11-25T03:37:15,161 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/8a2954eabeb94f54a9371b30bc7244f9 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/8a2954eabeb94f54a9371b30bc7244f9 2024-11-25T03:37:15,162 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/55ea99d3fef445ba9751d5b90eb32cfc to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/55ea99d3fef445ba9751d5b90eb32cfc 2024-11-25T03:37:15,164 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/b7bd8e46c82e44e4889492c77808ec26 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/b7bd8e46c82e44e4889492c77808ec26 2024-11-25T03:37:15,165 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/f4eb4f81c84e4fc58b7b5afbc270aa99 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/f4eb4f81c84e4fc58b7b5afbc270aa99 2024-11-25T03:37:15,166 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/5fc6ff77e2ad4dc2bd5af8c06c608734 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/5fc6ff77e2ad4dc2bd5af8c06c608734 2024-11-25T03:37:15,167 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/75389c1961f844dfad5c8bb8d5f7c9c0 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/75389c1961f844dfad5c8bb8d5f7c9c0 2024-11-25T03:37:15,168 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/598260aa127c4d208a30e10c193a924b to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/598260aa127c4d208a30e10c193a924b 2024-11-25T03:37:15,169 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/ef7b5767eabd487ea1d47f718af9ef9b to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/ef7b5767eabd487ea1d47f718af9ef9b 2024-11-25T03:37:15,170 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/3a466108ba57487a813214b8bb4ac583 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/3a466108ba57487a813214b8bb4ac583 2024-11-25T03:37:15,171 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/4eda90d9af6c485b9c89ba8f5a2576c9 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/4eda90d9af6c485b9c89ba8f5a2576c9 2024-11-25T03:37:15,173 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/171def67df4047139f7eb7ecf821e3f7 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/171def67df4047139f7eb7ecf821e3f7 2024-11-25T03:37:15,174 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/2423ac6c69464abda0e01cdc5b80dc24 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/2423ac6c69464abda0e01cdc5b80dc24 2024-11-25T03:37:15,175 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/94d7916d53164884b61051bd06b2133c to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/94d7916d53164884b61051bd06b2133c 2024-11-25T03:37:15,176 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/5cda8c5ceee848b5b283395f3792d68e to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/5cda8c5ceee848b5b283395f3792d68e 2024-11-25T03:37:15,177 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/c2ab1f865f6d46bf8006b079c1734b73 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/c2ab1f865f6d46bf8006b079c1734b73 2024-11-25T03:37:15,178 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/703b445cc5e84eb1aee104369f0e9c16 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/703b445cc5e84eb1aee104369f0e9c16 2024-11-25T03:37:15,180 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/fbfed05dfb194d61868bb657295cf4f6 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/fbfed05dfb194d61868bb657295cf4f6 2024-11-25T03:37:15,194 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/6783223d439b44ab87c1b0b7c6149edc, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/b025414f7057477c8fc36a619ccafe9d, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/7dfc7fe3067d44e4a3c122615f020c60, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/a144230e0cee4cfba4b71b1755744ee1, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/7a1df036c47b402db6d29ea4340f178f, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/61e2ee120a214a968d0a8e8da7cde3e8, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/c9949a29bd76499c9751e8876d3fae83, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/ff94de45bc1846089a69d16458dfc752, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/ab4376d67d74440e931def5856999ff0, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/4b8dbae4870c4fc0890209df19998adf, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/33c399cae6a9488e891c133857143039, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/ea80579fde3f4ee0a5d7389807668626, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/98c7909d282f410da21ad762115f771b, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/8d4bbebed0324bcd9ddd2e014efec18b, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/2807a04f71d44f8abac2d2e5aa09e188, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/5386e50525b24a2d8840f20fad1aa80a, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/bded33507fa549c2bc29fc181238fb8f, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/cf3871be4b5143a790b5c145ddbb6f68, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/bb28620f0b45402c814d00b4dfd8e8e9, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/d11c828ca88c4102b7296fc77e81110c, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/b0b1b53635c644199191855c06e93e05, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/5ee84ec1183a4a28881e7a06fe28a4e4, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/fa56a78b8b534c1eafc7f6fc57e64537, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/3e190c38e4584194aeff476a7fa64f7d, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/557bc182dad544b4b44f0c4ec8e0b5b5, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/f7ff0625629b49ada76f03a080ece6db, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/8ff0a55df2ad4ff49ffb61591260b93d, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/213a1cc75d86424f84cba3390ede6a82, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/6dc568a4ad3b46b0888283e5faad0b62, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/cf65fa346d074c3b9fe2cc96750a0d78, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/02ed19764cfe41ff8482396eaf764b9e, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/7d522c1bf29e41f6a63aa0f7a4116a46, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/33a864f7480244329da1bc4f5e9152ff, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/c39f42fb50274769a53c0ff6dfcc0f3c, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/102ba5317af94e13b0280e1f5f827e93, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/b385b814e5ae4aeb8df60d18e5eca324, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/e2ad2634834744aebe6e1f3ea6ec9179] to archive 2024-11-25T03:37:15,195 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T03:37:15,197 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/6783223d439b44ab87c1b0b7c6149edc to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/6783223d439b44ab87c1b0b7c6149edc 2024-11-25T03:37:15,198 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/b025414f7057477c8fc36a619ccafe9d to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/b025414f7057477c8fc36a619ccafe9d 2024-11-25T03:37:15,199 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/7dfc7fe3067d44e4a3c122615f020c60 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/7dfc7fe3067d44e4a3c122615f020c60 2024-11-25T03:37:15,200 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/a144230e0cee4cfba4b71b1755744ee1 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/a144230e0cee4cfba4b71b1755744ee1 2024-11-25T03:37:15,201 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/7a1df036c47b402db6d29ea4340f178f to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/7a1df036c47b402db6d29ea4340f178f 2024-11-25T03:37:15,202 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/61e2ee120a214a968d0a8e8da7cde3e8 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/61e2ee120a214a968d0a8e8da7cde3e8 2024-11-25T03:37:15,204 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/c9949a29bd76499c9751e8876d3fae83 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/c9949a29bd76499c9751e8876d3fae83 2024-11-25T03:37:15,205 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/ff94de45bc1846089a69d16458dfc752 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/ff94de45bc1846089a69d16458dfc752 2024-11-25T03:37:15,206 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/ab4376d67d74440e931def5856999ff0 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/ab4376d67d74440e931def5856999ff0 2024-11-25T03:37:15,207 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/4b8dbae4870c4fc0890209df19998adf to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/4b8dbae4870c4fc0890209df19998adf 2024-11-25T03:37:15,209 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/33c399cae6a9488e891c133857143039 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/33c399cae6a9488e891c133857143039 2024-11-25T03:37:15,210 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/ea80579fde3f4ee0a5d7389807668626 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/ea80579fde3f4ee0a5d7389807668626 2024-11-25T03:37:15,211 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/98c7909d282f410da21ad762115f771b to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/98c7909d282f410da21ad762115f771b 2024-11-25T03:37:15,212 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/8d4bbebed0324bcd9ddd2e014efec18b to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/8d4bbebed0324bcd9ddd2e014efec18b 2024-11-25T03:37:15,213 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/2807a04f71d44f8abac2d2e5aa09e188 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/2807a04f71d44f8abac2d2e5aa09e188 2024-11-25T03:37:15,214 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/5386e50525b24a2d8840f20fad1aa80a to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/5386e50525b24a2d8840f20fad1aa80a 2024-11-25T03:37:15,215 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/bded33507fa549c2bc29fc181238fb8f to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/bded33507fa549c2bc29fc181238fb8f 2024-11-25T03:37:15,217 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/cf3871be4b5143a790b5c145ddbb6f68 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/cf3871be4b5143a790b5c145ddbb6f68 2024-11-25T03:37:15,218 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/bb28620f0b45402c814d00b4dfd8e8e9 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/bb28620f0b45402c814d00b4dfd8e8e9 2024-11-25T03:37:15,219 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/d11c828ca88c4102b7296fc77e81110c to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/d11c828ca88c4102b7296fc77e81110c 2024-11-25T03:37:15,221 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/b0b1b53635c644199191855c06e93e05 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/b0b1b53635c644199191855c06e93e05 2024-11-25T03:37:15,222 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/5ee84ec1183a4a28881e7a06fe28a4e4 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/5ee84ec1183a4a28881e7a06fe28a4e4 2024-11-25T03:37:15,224 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/fa56a78b8b534c1eafc7f6fc57e64537 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/fa56a78b8b534c1eafc7f6fc57e64537 2024-11-25T03:37:15,225 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/3e190c38e4584194aeff476a7fa64f7d to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/3e190c38e4584194aeff476a7fa64f7d 2024-11-25T03:37:15,226 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/557bc182dad544b4b44f0c4ec8e0b5b5 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/557bc182dad544b4b44f0c4ec8e0b5b5 2024-11-25T03:37:15,227 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/f7ff0625629b49ada76f03a080ece6db to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/f7ff0625629b49ada76f03a080ece6db 2024-11-25T03:37:15,228 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/8ff0a55df2ad4ff49ffb61591260b93d to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/8ff0a55df2ad4ff49ffb61591260b93d 2024-11-25T03:37:15,229 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/213a1cc75d86424f84cba3390ede6a82 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/213a1cc75d86424f84cba3390ede6a82 2024-11-25T03:37:15,230 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/6dc568a4ad3b46b0888283e5faad0b62 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/6dc568a4ad3b46b0888283e5faad0b62 2024-11-25T03:37:15,232 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/cf65fa346d074c3b9fe2cc96750a0d78 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/cf65fa346d074c3b9fe2cc96750a0d78 2024-11-25T03:37:15,233 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/02ed19764cfe41ff8482396eaf764b9e to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/02ed19764cfe41ff8482396eaf764b9e 2024-11-25T03:37:15,234 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/7d522c1bf29e41f6a63aa0f7a4116a46 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/7d522c1bf29e41f6a63aa0f7a4116a46 2024-11-25T03:37:15,235 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/33a864f7480244329da1bc4f5e9152ff to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/33a864f7480244329da1bc4f5e9152ff 2024-11-25T03:37:15,236 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/c39f42fb50274769a53c0ff6dfcc0f3c to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/c39f42fb50274769a53c0ff6dfcc0f3c 2024-11-25T03:37:15,237 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/102ba5317af94e13b0280e1f5f827e93 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/102ba5317af94e13b0280e1f5f827e93 2024-11-25T03:37:15,239 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/b385b814e5ae4aeb8df60d18e5eca324 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/b385b814e5ae4aeb8df60d18e5eca324 2024-11-25T03:37:15,240 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/e2ad2634834744aebe6e1f3ea6ec9179 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/e2ad2634834744aebe6e1f3ea6ec9179 2024-11-25T03:37:15,241 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/8140caf5e34d4693bd1f103cacf50b6e, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/3eed32b972b6499599061d1e213b8b08, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/ae61d993d0f044b79e81ee41db2933cb, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/f911e024602b44f391a1abdfb25e2900, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/59c9fa86e7394eef8e0768ba0efb37d7, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/4deb5dcde47947b9b8479b16dc3497ff, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/d2defad255684ece8fd2d6f5d9d07a17, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/cbcaaad6dfc84fdc8930a6216e49528d, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/597414e26bc34972a20801a5906dddf0, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/196dd33618d446adbd71ad3c519dc395, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/dba3bc2703354402999a5d7da99de637, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/629076da23744d77989150735e5ddb3d, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/30543d4e604d4986bd606b6888865203, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/b1cdc4c3a05f4146844fc959665706b8, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/3ed4b6443af54090a00862dd9df90b4b, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/9b56b55cb36544ddbd3630f1849197e1, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/b8c0eebc0a774bdd9fc502ab35e9f9d2, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/efdd29c235294d5e840eef87e03b46c0, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/25ead75a50df4ca3bad2d9419428d210, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/4053d9dcb7894e10b81e9d90d84105ab, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/6aeea4ea4e1e470294f5ae9068f306cc, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/c1aafcdcc21344cba885d609b182343f, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/36f073135c5943eeb2dbcbdb367eb28d, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/6759dc9d7971404196ee7e1ce0aee090, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/a6c3686fd0e64a5c8b54613daafef776, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/4e2b41ce677f4ae1b40f499987a89e6f, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/0081d65d2a1545a29fad9060968af23b, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/6b7133ab3f214cf9ba9c0caf9b24779d, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/2fcb049491dd435daf53b987e929e961, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/acfe6dba1eb8466dbc8a29eb0f263458, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/55907ddbac2b421ca71e788a0cef9f9a, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/50a03085ddf345979d708a7008fef6cf, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/bd498a2839da4ba59451ec9b711c741d, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/04b46cd6845e4de99bab3557d5dab8bf, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/b913421b75184dcf9663ddd968b832eb, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/2c2103d741874c6e825a56c4b0f4f4c9] to archive 2024-11-25T03:37:15,242 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T03:37:15,244 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/8140caf5e34d4693bd1f103cacf50b6e to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/8140caf5e34d4693bd1f103cacf50b6e 2024-11-25T03:37:15,248 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/3eed32b972b6499599061d1e213b8b08 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/3eed32b972b6499599061d1e213b8b08 2024-11-25T03:37:15,250 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/ae61d993d0f044b79e81ee41db2933cb to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/ae61d993d0f044b79e81ee41db2933cb 2024-11-25T03:37:15,251 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/f911e024602b44f391a1abdfb25e2900 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/f911e024602b44f391a1abdfb25e2900 2024-11-25T03:37:15,252 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/59c9fa86e7394eef8e0768ba0efb37d7 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/59c9fa86e7394eef8e0768ba0efb37d7 2024-11-25T03:37:15,254 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/4deb5dcde47947b9b8479b16dc3497ff to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/4deb5dcde47947b9b8479b16dc3497ff 2024-11-25T03:37:15,255 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/d2defad255684ece8fd2d6f5d9d07a17 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/d2defad255684ece8fd2d6f5d9d07a17 2024-11-25T03:37:15,256 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/cbcaaad6dfc84fdc8930a6216e49528d to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/cbcaaad6dfc84fdc8930a6216e49528d 2024-11-25T03:37:15,257 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/597414e26bc34972a20801a5906dddf0 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/597414e26bc34972a20801a5906dddf0 2024-11-25T03:37:15,259 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/196dd33618d446adbd71ad3c519dc395 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/196dd33618d446adbd71ad3c519dc395 2024-11-25T03:37:15,260 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/dba3bc2703354402999a5d7da99de637 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/dba3bc2703354402999a5d7da99de637 2024-11-25T03:37:15,261 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/629076da23744d77989150735e5ddb3d to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/629076da23744d77989150735e5ddb3d 2024-11-25T03:37:15,262 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/30543d4e604d4986bd606b6888865203 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/30543d4e604d4986bd606b6888865203 2024-11-25T03:37:15,263 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/b1cdc4c3a05f4146844fc959665706b8 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/b1cdc4c3a05f4146844fc959665706b8 2024-11-25T03:37:15,264 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/3ed4b6443af54090a00862dd9df90b4b to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/3ed4b6443af54090a00862dd9df90b4b 2024-11-25T03:37:15,265 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/9b56b55cb36544ddbd3630f1849197e1 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/9b56b55cb36544ddbd3630f1849197e1 2024-11-25T03:37:15,266 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/b8c0eebc0a774bdd9fc502ab35e9f9d2 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/b8c0eebc0a774bdd9fc502ab35e9f9d2 2024-11-25T03:37:15,268 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/efdd29c235294d5e840eef87e03b46c0 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/efdd29c235294d5e840eef87e03b46c0 2024-11-25T03:37:15,269 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/25ead75a50df4ca3bad2d9419428d210 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/25ead75a50df4ca3bad2d9419428d210 2024-11-25T03:37:15,270 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/4053d9dcb7894e10b81e9d90d84105ab to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/4053d9dcb7894e10b81e9d90d84105ab 2024-11-25T03:37:15,271 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/6aeea4ea4e1e470294f5ae9068f306cc to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/6aeea4ea4e1e470294f5ae9068f306cc 2024-11-25T03:37:15,273 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/c1aafcdcc21344cba885d609b182343f to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/c1aafcdcc21344cba885d609b182343f 2024-11-25T03:37:15,274 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/36f073135c5943eeb2dbcbdb367eb28d to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/36f073135c5943eeb2dbcbdb367eb28d 2024-11-25T03:37:15,275 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/6759dc9d7971404196ee7e1ce0aee090 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/6759dc9d7971404196ee7e1ce0aee090 2024-11-25T03:37:15,276 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/a6c3686fd0e64a5c8b54613daafef776 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/a6c3686fd0e64a5c8b54613daafef776 2024-11-25T03:37:15,277 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/4e2b41ce677f4ae1b40f499987a89e6f to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/4e2b41ce677f4ae1b40f499987a89e6f 2024-11-25T03:37:15,279 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/0081d65d2a1545a29fad9060968af23b to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/0081d65d2a1545a29fad9060968af23b 2024-11-25T03:37:15,280 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/6b7133ab3f214cf9ba9c0caf9b24779d to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/6b7133ab3f214cf9ba9c0caf9b24779d 2024-11-25T03:37:15,281 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/2fcb049491dd435daf53b987e929e961 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/2fcb049491dd435daf53b987e929e961 2024-11-25T03:37:15,282 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/acfe6dba1eb8466dbc8a29eb0f263458 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/acfe6dba1eb8466dbc8a29eb0f263458 2024-11-25T03:37:15,283 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/55907ddbac2b421ca71e788a0cef9f9a to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/55907ddbac2b421ca71e788a0cef9f9a 2024-11-25T03:37:15,285 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/50a03085ddf345979d708a7008fef6cf to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/50a03085ddf345979d708a7008fef6cf 2024-11-25T03:37:15,286 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/bd498a2839da4ba59451ec9b711c741d to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/bd498a2839da4ba59451ec9b711c741d 2024-11-25T03:37:15,287 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/04b46cd6845e4de99bab3557d5dab8bf to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/04b46cd6845e4de99bab3557d5dab8bf 2024-11-25T03:37:15,289 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/b913421b75184dcf9663ddd968b832eb to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/b913421b75184dcf9663ddd968b832eb 2024-11-25T03:37:15,290 DEBUG [StoreCloser-TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/2c2103d741874c6e825a56c4b0f4f4c9 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/2c2103d741874c6e825a56c4b0f4f4c9 2024-11-25T03:37:15,297 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/recovered.edits/581.seqid, newMaxSeqId=581, maxSeqId=1 2024-11-25T03:37:15,298 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. 2024-11-25T03:37:15,298 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1676): Region close journal for ee186277170c2bdfb070a7c209286857: Waiting for close lock at 1732505833094Running coprocessor pre-close hooks at 1732505833094Disabling compacts and flushes for region at 1732505833094Disabling writes for close at 1732505833872 (+778 ms)Obtaining lock to block concurrent updates at 1732505833872Preparing flush snapshotting stores in ee186277170c2bdfb070a7c209286857 at 1732505833872Finished memstore snapshotting TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857., syncing WAL and waiting on mvcc, flushsize=dataSize=82440, getHeapSize=216720, getOffHeapSize=0, getCellsCount=1800 at 1732505833872Flushing stores of TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857. at 1732505833873 (+1 ms)Flushing ee186277170c2bdfb070a7c209286857/A: creating writer at 1732505833873Flushing ee186277170c2bdfb070a7c209286857/A: appending metadata at 1732505833875 (+2 ms)Flushing ee186277170c2bdfb070a7c209286857/A: closing flushed file at 1732505833875Flushing ee186277170c2bdfb070a7c209286857/B: creating writer at 1732505834285 (+410 ms)Flushing ee186277170c2bdfb070a7c209286857/B: appending metadata at 1732505834287 (+2 ms)Flushing ee186277170c2bdfb070a7c209286857/B: closing flushed file at 1732505834287Flushing ee186277170c2bdfb070a7c209286857/C: creating writer at 1732505834696 (+409 ms)Flushing ee186277170c2bdfb070a7c209286857/C: appending metadata at 1732505834699 (+3 ms)Flushing ee186277170c2bdfb070a7c209286857/C: closing flushed file at 1732505834699Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67d52f43: reopening flushed file at 1732505835109 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31ac97ea: reopening flushed file at 1732505835114 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@739c2e21: reopening flushed file at 1732505835120 (+6 ms)Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=0 B/0 for ee186277170c2bdfb070a7c209286857 in 1254ms, sequenceid=578, compaction requested=true at 1732505835126 (+6 ms)Writing region close event to WAL at 1732505835292 (+166 ms)Running coprocessor post-close hooks at 1732505835297 (+5 ms)Closed at 1732505835298 (+1 ms) 2024-11-25T03:37:15,301 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(157): Closed ee186277170c2bdfb070a7c209286857 2024-11-25T03:37:15,302 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=ee186277170c2bdfb070a7c209286857, regionState=CLOSED 2024-11-25T03:37:15,304 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 because future has completed 2024-11-25T03:37:15,306 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-25T03:37:15,306 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure ee186277170c2bdfb070a7c209286857, server=38a5d2077268,34881,1732505806588 in 2.3690 sec 2024-11-25T03:37:15,308 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-25T03:37:15,308 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee186277170c2bdfb070a7c209286857, UNASSIGN in 2.3750 sec 2024-11-25T03:37:15,311 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-25T03:37:15,311 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.3830 sec 2024-11-25T03:37:15,313 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732505835312"}]},"ts":"1732505835312"} 2024-11-25T03:37:15,314 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-25T03:37:15,314 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-25T03:37:15,317 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.3990 sec 2024-11-25T03:37:16,247 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-25T03:37:17,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-25T03:37:17,077 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:37:17,081 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-25T03:37:17,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:37:17,086 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=35, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:37:17,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-25T03:37:17,087 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=35, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:37:17,091 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857 2024-11-25T03:37:17,095 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A, FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B, FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C, FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/recovered.edits] 2024-11-25T03:37:17,098 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/36dec1ebe17a470cb74a18110105119f to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/36dec1ebe17a470cb74a18110105119f 2024-11-25T03:37:17,099 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/4541766953a64798b38dbf0bd4dd73c2 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/4541766953a64798b38dbf0bd4dd73c2 2024-11-25T03:37:17,100 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/4b06b2d2030b40a3b9e1f3fa5562ad46 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/4b06b2d2030b40a3b9e1f3fa5562ad46 2024-11-25T03:37:17,102 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/76e52d1d40d14ca494c0b96f94b912df to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/A/76e52d1d40d14ca494c0b96f94b912df 2024-11-25T03:37:17,104 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/2f72585f4138417794e2e427432544d6 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/2f72585f4138417794e2e427432544d6 2024-11-25T03:37:17,106 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/9859ed15e92547adbe0e7a2427cba252 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/9859ed15e92547adbe0e7a2427cba252 2024-11-25T03:37:17,107 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/b767147431ca4008bceba98b49cd95cf to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/b767147431ca4008bceba98b49cd95cf 2024-11-25T03:37:17,109 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/fd411f47f801470584324e3ae609fd49 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/B/fd411f47f801470584324e3ae609fd49 2024-11-25T03:37:17,111 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/2930a4bd6f5d405ca761b61cdf610e37 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/2930a4bd6f5d405ca761b61cdf610e37 2024-11-25T03:37:17,112 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/4a02fa5288de48b0a4f7ed0f6c4baddc to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/4a02fa5288de48b0a4f7ed0f6c4baddc 2024-11-25T03:37:17,113 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/a9cf3ea10111474d97bbe34abedfdb68 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/a9cf3ea10111474d97bbe34abedfdb68 2024-11-25T03:37:17,114 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/d3c793b0ad76436cbd157099b76bb484 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/C/d3c793b0ad76436cbd157099b76bb484 2024-11-25T03:37:17,117 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/recovered.edits/581.seqid to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857/recovered.edits/581.seqid 2024-11-25T03:37:17,118 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/ee186277170c2bdfb070a7c209286857 2024-11-25T03:37:17,118 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-25T03:37:17,123 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=35, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:37:17,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-25T03:37:17,133 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-25T03:37:17,163 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-25T03:37:17,164 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=35, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:37:17,164 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-25T03:37:17,165 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732505837164"}]},"ts":"9223372036854775807"} 2024-11-25T03:37:17,169 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-25T03:37:17,169 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ee186277170c2bdfb070a7c209286857, NAME => 'TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857.', STARTKEY => '', ENDKEY => ''}] 2024-11-25T03:37:17,169 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-25T03:37:17,170 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732505837169"}]},"ts":"9223372036854775807"} 2024-11-25T03:37:17,172 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-25T03:37:17,173 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=35, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:37:17,175 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 92 msec 2024-11-25T03:37:17,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-25T03:37:17,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-25T03:37:17,198 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:37:17,211 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=239 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1645846239_22 at /127.0.0.1:50096 [Waiting for operation #137] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;38a5d2077268:34881-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-798734266_22 at /127.0.0.1:47668 [Waiting for operation #312] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/38a5d2077268:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1645846239_22 at /127.0.0.1:47318 [Waiting for operation #318] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=348 (was 193) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6073 (was 6630) 2024-11-25T03:37:17,220 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=239, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=348, ProcessCount=11, AvailableMemoryMB=6073 2024-11-25T03:37:17,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T03:37:17,222 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T03:37:17,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-25T03:37:17,224 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T03:37:17,224 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:37:17,224 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 36 2024-11-25T03:37:17,225 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T03:37:17,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-25T03:37:17,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741959_1135 (size=959) 2024-11-25T03:37:17,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-25T03:37:17,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-25T03:37:17,635 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3 2024-11-25T03:37:17,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741960_1136 (size=53) 2024-11-25T03:37:17,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-25T03:37:18,043 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:37:18,043 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 9e284372599441cd748eba8961521ba1, disabling compactions & flushes 2024-11-25T03:37:18,043 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:18,043 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:18,043 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. after waiting 0 ms 2024-11-25T03:37:18,043 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:18,043 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:18,043 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9e284372599441cd748eba8961521ba1: Waiting for close lock at 1732505838043Disabling compacts and flushes for region at 1732505838043Disabling writes for close at 1732505838043Writing region close event to WAL at 1732505838043Closed at 1732505838043 2024-11-25T03:37:18,044 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T03:37:18,045 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732505838044"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732505838044"}]},"ts":"1732505838044"} 2024-11-25T03:37:18,047 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T03:37:18,047 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T03:37:18,048 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732505838048"}]},"ts":"1732505838048"} 2024-11-25T03:37:18,049 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-25T03:37:18,050 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e284372599441cd748eba8961521ba1, ASSIGN}] 2024-11-25T03:37:18,051 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e284372599441cd748eba8961521ba1, ASSIGN 2024-11-25T03:37:18,052 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e284372599441cd748eba8961521ba1, ASSIGN; state=OFFLINE, location=38a5d2077268,34881,1732505806588; forceNewPlan=false, retain=false 2024-11-25T03:37:18,202 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=9e284372599441cd748eba8961521ba1, regionState=OPENING, regionLocation=38a5d2077268,34881,1732505806588 2024-11-25T03:37:18,205 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e284372599441cd748eba8961521ba1, ASSIGN because future has completed 2024-11-25T03:37:18,205 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588}] 2024-11-25T03:37:18,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-25T03:37:18,362 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:18,362 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:37:18,362 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9e284372599441cd748eba8961521ba1 2024-11-25T03:37:18,362 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:37:18,362 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for 9e284372599441cd748eba8961521ba1 2024-11-25T03:37:18,362 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for 9e284372599441cd748eba8961521ba1 2024-11-25T03:37:18,364 INFO [StoreOpener-9e284372599441cd748eba8961521ba1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9e284372599441cd748eba8961521ba1 2024-11-25T03:37:18,366 INFO [StoreOpener-9e284372599441cd748eba8961521ba1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9e284372599441cd748eba8961521ba1 columnFamilyName A 2024-11-25T03:37:18,366 DEBUG [StoreOpener-9e284372599441cd748eba8961521ba1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:37:18,366 INFO [StoreOpener-9e284372599441cd748eba8961521ba1-1 {}] regionserver.HStore(327): Store=9e284372599441cd748eba8961521ba1/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:37:18,367 INFO [StoreOpener-9e284372599441cd748eba8961521ba1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9e284372599441cd748eba8961521ba1 2024-11-25T03:37:18,368 INFO [StoreOpener-9e284372599441cd748eba8961521ba1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9e284372599441cd748eba8961521ba1 columnFamilyName B 2024-11-25T03:37:18,368 DEBUG [StoreOpener-9e284372599441cd748eba8961521ba1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:37:18,368 INFO [StoreOpener-9e284372599441cd748eba8961521ba1-1 {}] regionserver.HStore(327): Store=9e284372599441cd748eba8961521ba1/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:37:18,368 INFO [StoreOpener-9e284372599441cd748eba8961521ba1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9e284372599441cd748eba8961521ba1 2024-11-25T03:37:18,370 INFO [StoreOpener-9e284372599441cd748eba8961521ba1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9e284372599441cd748eba8961521ba1 columnFamilyName C 2024-11-25T03:37:18,370 DEBUG [StoreOpener-9e284372599441cd748eba8961521ba1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:37:18,370 INFO [StoreOpener-9e284372599441cd748eba8961521ba1-1 {}] regionserver.HStore(327): Store=9e284372599441cd748eba8961521ba1/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:37:18,370 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for 9e284372599441cd748eba8961521ba1 2024-11-25T03:37:18,371 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1 2024-11-25T03:37:18,371 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1 2024-11-25T03:37:18,372 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for 9e284372599441cd748eba8961521ba1 2024-11-25T03:37:18,372 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for 9e284372599441cd748eba8961521ba1 2024-11-25T03:37:18,373 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T03:37:18,374 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for 9e284372599441cd748eba8961521ba1 2024-11-25T03:37:18,376 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:37:18,377 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened 9e284372599441cd748eba8961521ba1; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64604231, jitterRate=-0.03732194006443024}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T03:37:18,377 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9e284372599441cd748eba8961521ba1 2024-11-25T03:37:18,378 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for 9e284372599441cd748eba8961521ba1: Running coprocessor pre-open hook at 1732505838363Writing region info on filesystem at 1732505838363Initializing all the Stores at 1732505838363Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732505838363Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732505838364 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732505838364Cleaning up temporary data from old regions at 1732505838372 (+8 ms)Running coprocessor post-open hooks at 1732505838377 (+5 ms)Region opened successfully at 1732505838378 (+1 ms) 2024-11-25T03:37:18,379 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., pid=38, masterSystemTime=1732505838357 2024-11-25T03:37:18,381 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:18,381 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:18,382 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=9e284372599441cd748eba8961521ba1, regionState=OPEN, openSeqNum=2, regionLocation=38a5d2077268,34881,1732505806588 2024-11-25T03:37:18,383 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 because future has completed 2024-11-25T03:37:18,386 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-25T03:37:18,386 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure 9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 in 179 msec 2024-11-25T03:37:18,388 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-25T03:37:18,388 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e284372599441cd748eba8961521ba1, ASSIGN in 337 msec 2024-11-25T03:37:18,389 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T03:37:18,389 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732505838389"}]},"ts":"1732505838389"} 2024-11-25T03:37:18,391 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-25T03:37:18,392 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T03:37:18,393 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1700 sec 2024-11-25T03:37:19,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-25T03:37:19,368 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:37:19,369 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79527845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:19,369 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:37:19,370 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:37:19,371 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:37:19,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:37:19,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:37:19,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a746049, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:19,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:37:19,373 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:37:19,373 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:19,374 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51594, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:37:19,374 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b76b47a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:19,375 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:37:19,376 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:37:19,376 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:37:19,377 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52972, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:37:19,379 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T03:37:19,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 38a5d2077268,42243,1732505805780 2024-11-25T03:37:19,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@73b9f97d 2024-11-25T03:37:19,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T03:37:19,381 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51602, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T03:37:19,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T03:37:19,386 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T03:37:19,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=39, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-25T03:37:19,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741961_1137 (size=995) 2024-11-25T03:37:19,805 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-25T03:37:19,805 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-25T03:37:19,809 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T03:37:19,815 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e284372599441cd748eba8961521ba1, REOPEN/MOVE}] 2024-11-25T03:37:19,816 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e284372599441cd748eba8961521ba1, REOPEN/MOVE 2024-11-25T03:37:19,817 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=9e284372599441cd748eba8961521ba1, regionState=CLOSING, regionLocation=38a5d2077268,34881,1732505806588 2024-11-25T03:37:19,819 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e284372599441cd748eba8961521ba1, REOPEN/MOVE because future has completed 2024-11-25T03:37:19,819 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T03:37:19,819 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=41, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588}] 2024-11-25T03:37:19,972 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] handler.UnassignRegionHandler(122): Close 9e284372599441cd748eba8961521ba1 2024-11-25T03:37:19,972 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T03:37:19,972 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1722): Closing 9e284372599441cd748eba8961521ba1, disabling compactions & flushes 2024-11-25T03:37:19,972 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:19,972 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:19,972 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. after waiting 0 ms 2024-11-25T03:37:19,972 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:19,978 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-25T03:37:19,979 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:19,979 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1676): Region close journal for 9e284372599441cd748eba8961521ba1: Waiting for close lock at 1732505839972Running coprocessor pre-close hooks at 1732505839972Disabling compacts and flushes for region at 1732505839972Disabling writes for close at 1732505839972Writing region close event to WAL at 1732505839973 (+1 ms)Running coprocessor post-close hooks at 1732505839979 (+6 ms)Closed at 1732505839979 2024-11-25T03:37:19,979 WARN [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegionServer(3299): Not adding moved region record: 9e284372599441cd748eba8961521ba1 to self. 2024-11-25T03:37:19,981 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] handler.UnassignRegionHandler(157): Closed 9e284372599441cd748eba8961521ba1 2024-11-25T03:37:19,982 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=9e284372599441cd748eba8961521ba1, regionState=CLOSED 2024-11-25T03:37:19,983 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=41, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 because future has completed 2024-11-25T03:37:19,986 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=41 2024-11-25T03:37:19,987 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=41, state=SUCCESS, hasLock=false; CloseRegionProcedure 9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 in 166 msec 2024-11-25T03:37:19,987 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e284372599441cd748eba8961521ba1, REOPEN/MOVE; state=CLOSED, location=38a5d2077268,34881,1732505806588; forceNewPlan=false, retain=true 2024-11-25T03:37:20,138 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=9e284372599441cd748eba8961521ba1, regionState=OPENING, regionLocation=38a5d2077268,34881,1732505806588 2024-11-25T03:37:20,140 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e284372599441cd748eba8961521ba1, REOPEN/MOVE because future has completed 2024-11-25T03:37:20,140 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=41, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588}] 2024-11-25T03:37:20,295 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:20,295 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:37:20,296 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9e284372599441cd748eba8961521ba1 2024-11-25T03:37:20,296 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:37:20,296 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for 9e284372599441cd748eba8961521ba1 2024-11-25T03:37:20,296 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for 9e284372599441cd748eba8961521ba1 2024-11-25T03:37:20,301 INFO [StoreOpener-9e284372599441cd748eba8961521ba1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9e284372599441cd748eba8961521ba1 2024-11-25T03:37:20,308 INFO [StoreOpener-9e284372599441cd748eba8961521ba1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9e284372599441cd748eba8961521ba1 columnFamilyName A 2024-11-25T03:37:20,310 DEBUG [StoreOpener-9e284372599441cd748eba8961521ba1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:37:20,311 INFO [StoreOpener-9e284372599441cd748eba8961521ba1-1 {}] regionserver.HStore(327): Store=9e284372599441cd748eba8961521ba1/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:37:20,311 INFO [StoreOpener-9e284372599441cd748eba8961521ba1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9e284372599441cd748eba8961521ba1 2024-11-25T03:37:20,312 INFO [StoreOpener-9e284372599441cd748eba8961521ba1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9e284372599441cd748eba8961521ba1 columnFamilyName B 2024-11-25T03:37:20,312 DEBUG [StoreOpener-9e284372599441cd748eba8961521ba1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:37:20,313 INFO [StoreOpener-9e284372599441cd748eba8961521ba1-1 {}] regionserver.HStore(327): Store=9e284372599441cd748eba8961521ba1/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:37:20,313 INFO [StoreOpener-9e284372599441cd748eba8961521ba1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9e284372599441cd748eba8961521ba1 2024-11-25T03:37:20,314 INFO [StoreOpener-9e284372599441cd748eba8961521ba1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9e284372599441cd748eba8961521ba1 columnFamilyName C 2024-11-25T03:37:20,314 DEBUG [StoreOpener-9e284372599441cd748eba8961521ba1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:37:20,315 INFO [StoreOpener-9e284372599441cd748eba8961521ba1-1 {}] regionserver.HStore(327): Store=9e284372599441cd748eba8961521ba1/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:37:20,315 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for 9e284372599441cd748eba8961521ba1 2024-11-25T03:37:20,315 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1 2024-11-25T03:37:20,316 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1 2024-11-25T03:37:20,317 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for 9e284372599441cd748eba8961521ba1 2024-11-25T03:37:20,317 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for 9e284372599441cd748eba8961521ba1 2024-11-25T03:37:20,318 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T03:37:20,319 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for 9e284372599441cd748eba8961521ba1 2024-11-25T03:37:20,320 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened 9e284372599441cd748eba8961521ba1; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67091672, jitterRate=-2.561807632446289E-4}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T03:37:20,320 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9e284372599441cd748eba8961521ba1 2024-11-25T03:37:20,321 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for 9e284372599441cd748eba8961521ba1: Running coprocessor pre-open hook at 1732505840296Writing region info on filesystem at 1732505840296Initializing all the Stores at 1732505840297 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732505840297Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732505840297Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732505840297Cleaning up temporary data from old regions at 1732505840317 (+20 ms)Running coprocessor post-open hooks at 1732505840320 (+3 ms)Region opened successfully at 1732505840321 (+1 ms) 2024-11-25T03:37:20,322 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., pid=43, masterSystemTime=1732505840292 2024-11-25T03:37:20,325 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:20,325 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:20,325 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=9e284372599441cd748eba8961521ba1, regionState=OPEN, openSeqNum=5, regionLocation=38a5d2077268,34881,1732505806588 2024-11-25T03:37:20,328 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=41, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 because future has completed 2024-11-25T03:37:20,338 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=41 2024-11-25T03:37:20,338 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=41, state=SUCCESS, hasLock=false; OpenRegionProcedure 9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 in 189 msec 2024-11-25T03:37:20,340 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-25T03:37:20,340 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e284372599441cd748eba8961521ba1, REOPEN/MOVE in 523 msec 2024-11-25T03:37:20,343 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-25T03:37:20,343 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 533 msec 2024-11-25T03:37:20,347 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 956 msec 2024-11-25T03:37:20,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-25T03:37:20,349 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:37:20,349 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5adc4e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:20,349 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:37:20,349 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:37:20,351 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:37:20,351 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:37:20,351 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:37:20,351 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a60f6c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:20,351 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:37:20,351 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:37:20,352 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:20,353 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51626, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:37:20,354 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@799faa56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:20,354 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@681c9b14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:20,354 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:37:20,355 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:37:20,355 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:37:20,355 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:37:20,356 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:37:20,356 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d758f02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:20,356 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:37:20,356 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:37:20,356 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:20,357 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:37:20,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ff46a73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:20,358 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51d91e8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:20,358 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:37:20,358 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:37:20,359 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:37:20,359 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:37:20,359 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:37:20,359 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@785a2fbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:20,359 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:37:20,360 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:37:20,360 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:20,360 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51666, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:37:20,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@556d05d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:20,362 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@771f5e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:20,362 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:37:20,362 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:37:20,363 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:37:20,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:37:20,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:37:20,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37b661fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:20,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:37:20,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:37:20,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:20,365 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:37:20,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72d4bac8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:20,366 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37b75e2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:20,366 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:37:20,366 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:37:20,367 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:37:20,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:37:20,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:37:20,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d50f08d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:20,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:37:20,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:37:20,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:20,369 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51720, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:37:20,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12bc3d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:20,376 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f5453a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:20,376 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:37:20,376 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:37:20,377 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:37:20,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:37:20,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:37:20,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17b0504c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:20,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:37:20,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:37:20,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:20,378 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51736, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:37:20,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68cb4051, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:20,380 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73b6e690, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:20,380 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:37:20,380 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:37:20,380 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:37:20,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:37:20,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:37:20,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab9a88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:20,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:37:20,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:37:20,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:20,382 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51748, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:37:20,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3202d9a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:20,383 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bd05ea8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:20,383 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:37:20,383 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:37:20,384 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:37:20,384 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:37:20,384 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:37:20,384 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25c0cc05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:20,384 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:37:20,385 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:37:20,385 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:20,385 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51776, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:37:20,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0dd047, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:20,386 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dff183a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:20,386 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:37:20,387 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:37:20,387 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:37:20,387 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:37:20,387 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:37:20,387 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21f50e09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:20,388 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:37:20,388 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:37:20,388 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:20,388 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51792, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:37:20,389 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@364eb416, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:20,393 DEBUG [Thread-684 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:37:20,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:37:20,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:37:20,395 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52976, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:37:20,397 DEBUG [Thread-686 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:37:20,397 DEBUG [Thread-695 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:37:20,398 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:37:20,398 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5] 2024-11-25T03:37:20,399 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:37:20,399 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:37:20,400 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52978, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:37:20,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:37:20,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:37:20,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-25T03:37:20,403 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52982, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:37:20,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5] 2024-11-25T03:37:20,404 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:37:20,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-25T03:37:20,407 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:37:20,408 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:37:20,409 DEBUG [Thread-691 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:37:20,409 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:37:20,410 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5] 2024-11-25T03:37:20,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:37:20,412 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52996, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:37:20,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5] 2024-11-25T03:37:20,421 DEBUG [Thread-689 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:37:20,423 DEBUG [Thread-689 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:37:20,427 DEBUG [Thread-689 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:37:20,429 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53002, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:37:20,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5] 2024-11-25T03:37:20,437 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:37:20,437 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e284372599441cd748eba8961521ba1 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-25T03:37:20,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 9e284372599441cd748eba8961521ba1 2024-11-25T03:37:20,444 DEBUG [Thread-678 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:37:20,449 DEBUG [Thread-680 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:37:20,450 DEBUG [Thread-682 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:37:20,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:37:20,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:37:20,451 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:37:20,451 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:37:20,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:37:20,452 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:37:20,452 DEBUG [Thread-693 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:37:20,452 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53018, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:37:20,453 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53024, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:37:20,453 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53032, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:37:20,454 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:37:20,454 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:37:20,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5] 2024-11-25T03:37:20,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5] 2024-11-25T03:37:20,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5] 2024-11-25T03:37:20,456 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53046, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:37:20,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5] 2024-11-25T03:37:20,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411255ec17bc38d0242c3bb0bb0b918920265_9e284372599441cd748eba8961521ba1 is 50, key is test_row_0/A:col10/1732505840406/Put/seqid=0 2024-11-25T03:37:20,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741962_1138 (size=17034) 2024-11-25T03:37:20,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-25T03:37:20,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-25T03:37:20,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:20,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. as already flushing 2024-11-25T03:37:20,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:20,564 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:20,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:20,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:20,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:20,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505900562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:20,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:20,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:20,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:20,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:20,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505900560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:20,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:20,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:20,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:20,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:20,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1732505900575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:20,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:20,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:20,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:20,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:20,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505900575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:20,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:20,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:20,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:20,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:20,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505900577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:20,583 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:20,583 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:20,583 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:20,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:20,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505900688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:20,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:20,694 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:20,694 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:20,694 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:20,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505900688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:20,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:20,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:20,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:20,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:20,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505900689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:20,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:20,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1732505900689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:20,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:20,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:20,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505900689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:20,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:20,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:20,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:20,700 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:20,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:20,700 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:20,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:20,700 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:20,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-25T03:37:20,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:20,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. as already flushing 2024-11-25T03:37:20,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:20,719 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:20,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:20,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:20,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-25T03:37:20,828 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T03:37:20,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-25T03:37:20,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:20,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. as already flushing 2024-11-25T03:37:20,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:20,874 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:20,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:20,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:20,899 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:37:20,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:20,905 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411255ec17bc38d0242c3bb0bb0b918920265_9e284372599441cd748eba8961521ba1 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411255ec17bc38d0242c3bb0bb0b918920265_9e284372599441cd748eba8961521ba1 2024-11-25T03:37:20,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505900898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:20,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:20,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:20,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:20,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:20,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505900898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:20,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:20,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:20,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:20,907 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/5b65eee8537f48e585855082a70a8cca, store: [table=TestAcidGuarantees family=A region=9e284372599441cd748eba8961521ba1] 2024-11-25T03:37:20,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:20,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505900910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:20,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:20,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:20,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:20,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:20,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505900910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:20,915 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:20,915 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:20,915 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:20,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/5b65eee8537f48e585855082a70a8cca is 175, key is test_row_0/A:col10/1732505840406/Put/seqid=0 2024-11-25T03:37:20,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:20,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1732505900910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:20,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:20,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:20,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:20,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741963_1139 (size=48139) 2024-11-25T03:37:20,939 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/5b65eee8537f48e585855082a70a8cca 2024-11-25T03:37:20,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/1c872f79dc9843d8bc8b160f29350b47 is 50, key is test_row_0/B:col10/1732505840406/Put/seqid=0 2024-11-25T03:37:21,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741964_1140 (size=12001) 2024-11-25T03:37:21,004 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/1c872f79dc9843d8bc8b160f29350b47 2024-11-25T03:37:21,027 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-25T03:37:21,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:21,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. as already flushing 2024-11-25T03:37:21,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:21,028 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:21,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:21,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:21,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-25T03:37:21,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/d0df032432bc4dd0925f85a5913c5295 is 50, key is test_row_0/C:col10/1732505840406/Put/seqid=0 2024-11-25T03:37:21,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741965_1141 (size=12001) 2024-11-25T03:37:21,093 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/d0df032432bc4dd0925f85a5913c5295 2024-11-25T03:37:21,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/5b65eee8537f48e585855082a70a8cca as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/5b65eee8537f48e585855082a70a8cca 2024-11-25T03:37:21,112 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/5b65eee8537f48e585855082a70a8cca, entries=250, sequenceid=15, filesize=47.0 K 2024-11-25T03:37:21,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/1c872f79dc9843d8bc8b160f29350b47 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/1c872f79dc9843d8bc8b160f29350b47 2024-11-25T03:37:21,122 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/1c872f79dc9843d8bc8b160f29350b47, entries=150, sequenceid=15, filesize=11.7 K 2024-11-25T03:37:21,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/d0df032432bc4dd0925f85a5913c5295 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/d0df032432bc4dd0925f85a5913c5295 2024-11-25T03:37:21,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/d0df032432bc4dd0925f85a5913c5295, entries=150, sequenceid=15, filesize=11.7 K 2024-11-25T03:37:21,134 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 9e284372599441cd748eba8961521ba1 in 697ms, sequenceid=15, compaction requested=false 2024-11-25T03:37:21,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e284372599441cd748eba8961521ba1: 2024-11-25T03:37:21,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-25T03:37:21,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:21,183 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 9e284372599441cd748eba8961521ba1 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-25T03:37:21,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 9e284372599441cd748eba8961521ba1 2024-11-25T03:37:21,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. as already flushing 2024-11-25T03:37:21,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125f13bd85367c645d9bb506b75444f2660_9e284372599441cd748eba8961521ba1 is 50, key is test_row_0/A:col10/1732505840572/Put/seqid=0 2024-11-25T03:37:21,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:21,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1732505901237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:21,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:21,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:21,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:21,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741966_1142 (size=12154) 2024-11-25T03:37:21,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:21,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505901243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:21,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:21,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505901244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:21,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:21,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:21,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:21,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:21,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505901249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:21,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:21,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:21,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:21,260 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:21,260 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:21,260 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:21,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:21,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505901251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:21,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:21,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:21,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:21,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:21,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505901367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:21,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:21,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505901368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:21,373 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:21,373 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:21,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:21,373 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:21,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505901368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:21,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:21,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505901368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:21,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:21,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:21,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:21,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:21,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:21,374 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:21,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:21,374 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:21,374 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:21,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-25T03:37:21,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:21,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505901578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:21,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:21,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:21,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505901578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:21,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505901578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:21,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:21,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505901578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:21,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:21,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:21,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:21,584 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:21,584 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:21,584 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:21,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:21,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:21,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:21,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:21,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:21,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:21,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:37:21,672 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125f13bd85367c645d9bb506b75444f2660_9e284372599441cd748eba8961521ba1 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125f13bd85367c645d9bb506b75444f2660_9e284372599441cd748eba8961521ba1 2024-11-25T03:37:21,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/2bb0e3d1f8ba49b7a4c1ee22369de4eb, store: [table=TestAcidGuarantees family=A region=9e284372599441cd748eba8961521ba1] 2024-11-25T03:37:21,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/2bb0e3d1f8ba49b7a4c1ee22369de4eb is 175, key is test_row_0/A:col10/1732505840572/Put/seqid=0 2024-11-25T03:37:21,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741967_1143 (size=30955) 2024-11-25T03:37:21,691 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/2bb0e3d1f8ba49b7a4c1ee22369de4eb 2024-11-25T03:37:21,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/e4c061a8a8dd4284af6ed27de860d8cb is 50, key is test_row_0/B:col10/1732505840572/Put/seqid=0 2024-11-25T03:37:21,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741968_1144 (size=12001) 2024-11-25T03:37:21,736 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/e4c061a8a8dd4284af6ed27de860d8cb 2024-11-25T03:37:21,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/c2e1e591785846e58646371118bbc2ae is 50, key is test_row_0/C:col10/1732505840572/Put/seqid=0 2024-11-25T03:37:21,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:21,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1732505901757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:21,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:21,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:21,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:21,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741969_1145 (size=12001) 2024-11-25T03:37:21,782 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/c2e1e591785846e58646371118bbc2ae 2024-11-25T03:37:21,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/2bb0e3d1f8ba49b7a4c1ee22369de4eb as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/2bb0e3d1f8ba49b7a4c1ee22369de4eb 2024-11-25T03:37:21,802 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/2bb0e3d1f8ba49b7a4c1ee22369de4eb, entries=150, sequenceid=40, filesize=30.2 K 2024-11-25T03:37:21,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/e4c061a8a8dd4284af6ed27de860d8cb as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/e4c061a8a8dd4284af6ed27de860d8cb 2024-11-25T03:37:21,813 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/e4c061a8a8dd4284af6ed27de860d8cb, entries=150, sequenceid=40, filesize=11.7 K 2024-11-25T03:37:21,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/c2e1e591785846e58646371118bbc2ae as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/c2e1e591785846e58646371118bbc2ae 2024-11-25T03:37:21,822 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/c2e1e591785846e58646371118bbc2ae, entries=150, sequenceid=40, filesize=11.7 K 2024-11-25T03:37:21,824 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 9e284372599441cd748eba8961521ba1 in 641ms, sequenceid=40, compaction requested=false 2024-11-25T03:37:21,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 9e284372599441cd748eba8961521ba1: 2024-11-25T03:37:21,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:21,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-25T03:37:21,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-25T03:37:21,830 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-25T03:37:21,830 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4200 sec 2024-11-25T03:37:21,833 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.4320 sec 2024-11-25T03:37:21,890 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:37:21,890 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e284372599441cd748eba8961521ba1 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-25T03:37:21,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 9e284372599441cd748eba8961521ba1 2024-11-25T03:37:21,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411256b5f066e9ef54681aa7c96dfabb8f846_9e284372599441cd748eba8961521ba1 is 50, key is test_row_0/A:col10/1732505841887/Put/seqid=0 2024-11-25T03:37:21,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741970_1146 (size=17034) 2024-11-25T03:37:21,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:21,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505901943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:21,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:21,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:21,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:21,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:21,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505901945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:21,951 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:21,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:21,951 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:21,952 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:21,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505901946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:21,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:21,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:21,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:21,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:21,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505901950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:21,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:21,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:21,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:22,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:22,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505902057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:22,062 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:22,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:22,062 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:22,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:22,062 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:22,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505902057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:22,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505902057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:22,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:22,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:22,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:22,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:22,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:22,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:22,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:22,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505902067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:22,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:22,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:22,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:22,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:22,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505902267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:22,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:22,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505902267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:22,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:22,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:22,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:22,269 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:22,269 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:22,269 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:22,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:22,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505902267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:22,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:22,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:22,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:22,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:22,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505902278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:22,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:22,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:22,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:22,341 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:37:22,346 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411256b5f066e9ef54681aa7c96dfabb8f846_9e284372599441cd748eba8961521ba1 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411256b5f066e9ef54681aa7c96dfabb8f846_9e284372599441cd748eba8961521ba1 2024-11-25T03:37:22,350 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/e88e735532e64424913b93dd5baf1730, store: [table=TestAcidGuarantees family=A region=9e284372599441cd748eba8961521ba1] 2024-11-25T03:37:22,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/e88e735532e64424913b93dd5baf1730 is 175, key is test_row_0/A:col10/1732505841887/Put/seqid=0 2024-11-25T03:37:22,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741971_1147 (size=48139) 2024-11-25T03:37:22,361 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/e88e735532e64424913b93dd5baf1730 2024-11-25T03:37:22,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/2c02bc8164c44021a1a6aded80c73958 is 50, key is test_row_0/B:col10/1732505841887/Put/seqid=0 2024-11-25T03:37:22,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741972_1148 (size=12001) 2024-11-25T03:37:22,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-25T03:37:22,567 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:37:22,568 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:37:22,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-25T03:37:22,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-25T03:37:22,570 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:37:22,571 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:37:22,571 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:37:22,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:22,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505902577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:22,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:22,578 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:22,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505902577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:22,578 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:22,578 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:22,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:22,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:22,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:22,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:22,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505902577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:22,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:22,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:22,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:22,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:22,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505902588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:22,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:22,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:22,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:22,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-25T03:37:22,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-25T03:37:22,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:22,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. as already flushing 2024-11-25T03:37:22,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:22,724 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:22,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:22,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:22,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1732505902768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:22,776 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,34881,1732505806588 for 'test_row_2' in 9e284372599441cd748eba8961521ba1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:37:22,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:22,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:22,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:22,786 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/2c02bc8164c44021a1a6aded80c73958 2024-11-25T03:37:22,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/178ae2e30f164492a5bcea1cd13092cc is 50, key is test_row_0/C:col10/1732505841887/Put/seqid=0 2024-11-25T03:37:22,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741973_1149 (size=12001) 2024-11-25T03:37:22,822 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/178ae2e30f164492a5bcea1cd13092cc 2024-11-25T03:37:22,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/e88e735532e64424913b93dd5baf1730 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/e88e735532e64424913b93dd5baf1730 2024-11-25T03:37:22,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/e88e735532e64424913b93dd5baf1730, entries=250, sequenceid=53, filesize=47.0 K 2024-11-25T03:37:22,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/2c02bc8164c44021a1a6aded80c73958 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/2c02bc8164c44021a1a6aded80c73958 2024-11-25T03:37:22,852 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/2c02bc8164c44021a1a6aded80c73958, entries=150, sequenceid=53, filesize=11.7 K 2024-11-25T03:37:22,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/178ae2e30f164492a5bcea1cd13092cc as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/178ae2e30f164492a5bcea1cd13092cc 2024-11-25T03:37:22,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/178ae2e30f164492a5bcea1cd13092cc, entries=150, sequenceid=53, filesize=11.7 K 2024-11-25T03:37:22,861 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 9e284372599441cd748eba8961521ba1 in 971ms, sequenceid=53, compaction requested=true 2024-11-25T03:37:22,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e284372599441cd748eba8961521ba1: 2024-11-25T03:37:22,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e284372599441cd748eba8961521ba1:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:37:22,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:22,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e284372599441cd748eba8961521ba1:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:37:22,861 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:37:22,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:37:22,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e284372599441cd748eba8961521ba1:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:37:22,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T03:37:22,862 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:37:22,863 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:37:22,863 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): 9e284372599441cd748eba8961521ba1/B is initiating minor compaction (all files) 2024-11-25T03:37:22,863 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e284372599441cd748eba8961521ba1/B in TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:22,863 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/1c872f79dc9843d8bc8b160f29350b47, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/e4c061a8a8dd4284af6ed27de860d8cb, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/2c02bc8164c44021a1a6aded80c73958] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp, totalSize=35.2 K 2024-11-25T03:37:22,863 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127233 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:37:22,864 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 9e284372599441cd748eba8961521ba1/A is initiating minor compaction (all files) 2024-11-25T03:37:22,864 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e284372599441cd748eba8961521ba1/A in TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:22,864 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/5b65eee8537f48e585855082a70a8cca, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/2bb0e3d1f8ba49b7a4c1ee22369de4eb, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/e88e735532e64424913b93dd5baf1730] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp, totalSize=124.3 K 2024-11-25T03:37:22,864 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:22,864 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. files: [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/5b65eee8537f48e585855082a70a8cca, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/2bb0e3d1f8ba49b7a4c1ee22369de4eb, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/e88e735532e64424913b93dd5baf1730] 2024-11-25T03:37:22,864 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c872f79dc9843d8bc8b160f29350b47, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732505840406 2024-11-25T03:37:22,865 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4c061a8a8dd4284af6ed27de860d8cb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732505840570 2024-11-25T03:37:22,865 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b65eee8537f48e585855082a70a8cca, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732505840406 2024-11-25T03:37:22,866 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c02bc8164c44021a1a6aded80c73958, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732505841242 2024-11-25T03:37:22,868 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bb0e3d1f8ba49b7a4c1ee22369de4eb, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732505840570 2024-11-25T03:37:22,870 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting e88e735532e64424913b93dd5baf1730, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732505841239 2024-11-25T03:37:22,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-25T03:37:22,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:22,879 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 9e284372599441cd748eba8961521ba1 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-25T03:37:22,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-25T03:37:22,907 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e284372599441cd748eba8961521ba1#B#compaction#131 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:37:22,908 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/e71896520f18492fb2c734866bbb4e6c is 50, key is test_row_0/B:col10/1732505841887/Put/seqid=0 2024-11-25T03:37:22,913 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9e284372599441cd748eba8961521ba1] 2024-11-25T03:37:22,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125ed8433757f864a9f85d88e6b32dee154_9e284372599441cd748eba8961521ba1 is 50, key is test_row_0/A:col10/1732505841947/Put/seqid=0 2024-11-25T03:37:22,946 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125f35c50d6a27e4d44900007bee4a41b1b_9e284372599441cd748eba8961521ba1 store=[table=TestAcidGuarantees family=A region=9e284372599441cd748eba8961521ba1] 2024-11-25T03:37:22,954 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125f35c50d6a27e4d44900007bee4a41b1b_9e284372599441cd748eba8961521ba1, store=[table=TestAcidGuarantees family=A region=9e284372599441cd748eba8961521ba1] 2024-11-25T03:37:22,956 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125f35c50d6a27e4d44900007bee4a41b1b_9e284372599441cd748eba8961521ba1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9e284372599441cd748eba8961521ba1] 2024-11-25T03:37:22,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741974_1150 (size=12104) 2024-11-25T03:37:22,973 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/e71896520f18492fb2c734866bbb4e6c as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/e71896520f18492fb2c734866bbb4e6c 2024-11-25T03:37:22,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741976_1152 (size=4469) 2024-11-25T03:37:22,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741975_1151 (size=12154) 2024-11-25T03:37:22,987 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e284372599441cd748eba8961521ba1#A#compaction#132 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:37:22,988 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/e9f2be7b55f94605a4e4bd7f4a76296e is 175, key is test_row_0/A:col10/1732505841887/Put/seqid=0 2024-11-25T03:37:22,990 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e284372599441cd748eba8961521ba1/B of 9e284372599441cd748eba8961521ba1 into e71896520f18492fb2c734866bbb4e6c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:37:22,990 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e284372599441cd748eba8961521ba1: 2024-11-25T03:37:22,990 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., storeName=9e284372599441cd748eba8961521ba1/B, priority=13, startTime=1732505842861; duration=0sec 2024-11-25T03:37:22,990 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:37:22,991 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e284372599441cd748eba8961521ba1:B 2024-11-25T03:37:22,991 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:37:22,992 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:37:22,992 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): 9e284372599441cd748eba8961521ba1/C is initiating minor compaction (all files) 2024-11-25T03:37:22,992 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e284372599441cd748eba8961521ba1/C in TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:22,992 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/d0df032432bc4dd0925f85a5913c5295, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/c2e1e591785846e58646371118bbc2ae, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/178ae2e30f164492a5bcea1cd13092cc] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp, totalSize=35.2 K 2024-11-25T03:37:22,993 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0df032432bc4dd0925f85a5913c5295, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732505840406 2024-11-25T03:37:22,993 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2e1e591785846e58646371118bbc2ae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732505840570 2024-11-25T03:37:22,994 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 178ae2e30f164492a5bcea1cd13092cc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732505841242 2024-11-25T03:37:23,032 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e284372599441cd748eba8961521ba1#C#compaction#134 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:37:23,033 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/24f796b495bb410580e242aea444b720 is 50, key is test_row_0/C:col10/1732505841887/Put/seqid=0 2024-11-25T03:37:23,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741977_1153 (size=31058) 2024-11-25T03:37:23,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741978_1154 (size=12104) 2024-11-25T03:37:23,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 9e284372599441cd748eba8961521ba1 2024-11-25T03:37:23,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. as already flushing 2024-11-25T03:37:23,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:23,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:23,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505903112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:23,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505903112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:23,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:23,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:23,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:23,116 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:23,116 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:23,116 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:23,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:23,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505903116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:23,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:23,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505903120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:23,122 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:23,122 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:23,122 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:23,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:23,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:23,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:23,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-25T03:37:23,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:23,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505903217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:23,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:23,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:23,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:23,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:23,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505903227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:23,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:23,230 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:23,230 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:23,230 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:23,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505903228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:23,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:23,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:23,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505903228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:23,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:23,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:23,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:23,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:23,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:23,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:37:23,396 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125ed8433757f864a9f85d88e6b32dee154_9e284372599441cd748eba8961521ba1 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125ed8433757f864a9f85d88e6b32dee154_9e284372599441cd748eba8961521ba1 2024-11-25T03:37:23,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/353578fa21dc4a9cb549ae5e0108826f, store: [table=TestAcidGuarantees family=A region=9e284372599441cd748eba8961521ba1] 2024-11-25T03:37:23,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/353578fa21dc4a9cb549ae5e0108826f is 175, key is test_row_0/A:col10/1732505841947/Put/seqid=0 2024-11-25T03:37:23,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741979_1155 (size=30955) 2024-11-25T03:37:23,407 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=76, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/353578fa21dc4a9cb549ae5e0108826f 2024-11-25T03:37:23,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/2357accf81f04b9da6399c8a80f88e94 is 50, key is test_row_0/B:col10/1732505841947/Put/seqid=0 2024-11-25T03:37:23,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741980_1156 (size=12001) 2024-11-25T03:37:23,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:23,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505903438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:23,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:23,443 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:23,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505903438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:23,443 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:23,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:23,443 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:23,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505903438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:23,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:23,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505903438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:23,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:23,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:23,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:23,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:23,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:23,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:23,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:23,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:23,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:23,462 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/e9f2be7b55f94605a4e4bd7f4a76296e as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/e9f2be7b55f94605a4e4bd7f4a76296e 2024-11-25T03:37:23,470 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/24f796b495bb410580e242aea444b720 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/24f796b495bb410580e242aea444b720 2024-11-25T03:37:23,471 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e284372599441cd748eba8961521ba1/A of 9e284372599441cd748eba8961521ba1 into e9f2be7b55f94605a4e4bd7f4a76296e(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:37:23,471 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e284372599441cd748eba8961521ba1: 2024-11-25T03:37:23,471 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., storeName=9e284372599441cd748eba8961521ba1/A, priority=13, startTime=1732505842861; duration=0sec 2024-11-25T03:37:23,471 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:23,472 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e284372599441cd748eba8961521ba1:A 2024-11-25T03:37:23,478 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e284372599441cd748eba8961521ba1/C of 9e284372599441cd748eba8961521ba1 into 24f796b495bb410580e242aea444b720(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:37:23,478 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e284372599441cd748eba8961521ba1: 2024-11-25T03:37:23,478 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., storeName=9e284372599441cd748eba8961521ba1/C, priority=13, startTime=1732505842862; duration=0sec 2024-11-25T03:37:23,479 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:23,479 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e284372599441cd748eba8961521ba1:C 2024-11-25T03:37:23,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-25T03:37:23,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:23,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:23,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505903747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:23,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505903747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:23,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:23,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505903748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:23,751 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:23,751 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:23,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:23,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:23,751 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:23,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:23,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:23,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505903748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:23,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:23,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:23,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:23,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:23,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:23,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:23,833 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/2357accf81f04b9da6399c8a80f88e94 2024-11-25T03:37:23,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/aebda93c62b242a7a3fb8b403f59f0d0 is 50, key is test_row_0/C:col10/1732505841947/Put/seqid=0 2024-11-25T03:37:23,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741981_1157 (size=12001) 2024-11-25T03:37:23,855 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/aebda93c62b242a7a3fb8b403f59f0d0 2024-11-25T03:37:23,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/353578fa21dc4a9cb549ae5e0108826f as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/353578fa21dc4a9cb549ae5e0108826f 2024-11-25T03:37:23,867 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/353578fa21dc4a9cb549ae5e0108826f, entries=150, sequenceid=76, filesize=30.2 K 2024-11-25T03:37:23,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/2357accf81f04b9da6399c8a80f88e94 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/2357accf81f04b9da6399c8a80f88e94 2024-11-25T03:37:23,875 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/2357accf81f04b9da6399c8a80f88e94, entries=150, sequenceid=76, filesize=11.7 K 2024-11-25T03:37:23,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/aebda93c62b242a7a3fb8b403f59f0d0 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/aebda93c62b242a7a3fb8b403f59f0d0 2024-11-25T03:37:23,884 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/aebda93c62b242a7a3fb8b403f59f0d0, entries=150, sequenceid=76, filesize=11.7 K 2024-11-25T03:37:23,885 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 9e284372599441cd748eba8961521ba1 in 1006ms, sequenceid=76, compaction requested=false 2024-11-25T03:37:23,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 9e284372599441cd748eba8961521ba1: 2024-11-25T03:37:23,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:23,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-25T03:37:23,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-25T03:37:23,891 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-25T03:37:23,891 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3180 sec 2024-11-25T03:37:23,893 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.3240 sec 2024-11-25T03:37:24,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 9e284372599441cd748eba8961521ba1 2024-11-25T03:37:24,264 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:37:24,264 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e284372599441cd748eba8961521ba1 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-25T03:37:24,293 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411256be95ae7b7a1415393a9ad25e25ed3a3_9e284372599441cd748eba8961521ba1 is 50, key is test_row_0/A:col10/1732505843112/Put/seqid=0 2024-11-25T03:37:24,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:24,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505904295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:24,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:24,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:24,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505904301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:24,304 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:24,304 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:24,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:24,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:24,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505904304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:24,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505904308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:24,310 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:24,310 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:24,310 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:24,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:24,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:24,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:24,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:24,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:24,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:24,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741982_1158 (size=12154) 2024-11-25T03:37:24,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:24,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505904407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:24,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:24,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:24,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:24,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:24,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:24,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505904418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:24,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:24,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505904418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:24,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505904417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:24,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:24,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:24,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:24,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:24,427 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:24,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:24,427 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:24,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:24,427 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:24,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:24,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505904617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:24,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:24,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:24,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:24,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:24,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505904627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:24,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:24,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:24,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:24,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:24,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505904638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:24,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:24,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:24,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:24,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505904638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:24,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:24,641 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:24,641 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:24,641 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:24,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-25T03:37:24,718 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:37:24,718 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:37:24,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-25T03:37:24,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-25T03:37:24,721 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:37:24,722 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:37:24,722 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:37:24,728 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:37:24,733 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411256be95ae7b7a1415393a9ad25e25ed3a3_9e284372599441cd748eba8961521ba1 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411256be95ae7b7a1415393a9ad25e25ed3a3_9e284372599441cd748eba8961521ba1 2024-11-25T03:37:24,734 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/9d29a4d1d1f04166b8aec4f11a8ff5d8, store: [table=TestAcidGuarantees family=A region=9e284372599441cd748eba8961521ba1] 2024-11-25T03:37:24,736 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/9d29a4d1d1f04166b8aec4f11a8ff5d8 is 175, key is test_row_0/A:col10/1732505843112/Put/seqid=0 2024-11-25T03:37:24,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741983_1159 (size=30955) 2024-11-25T03:37:24,747 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/9d29a4d1d1f04166b8aec4f11a8ff5d8 2024-11-25T03:37:24,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/ab558a8ba57846bcad4896923a6392fc is 50, key is test_row_0/B:col10/1732505843112/Put/seqid=0 2024-11-25T03:37:24,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:24,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1732505904787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:24,794 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,34881,1732505806588 for 'test_row_2' in 9e284372599441cd748eba8961521ba1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:37:24,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:24,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:24,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:24,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741984_1160 (size=12001) 2024-11-25T03:37:24,823 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/ab558a8ba57846bcad4896923a6392fc 2024-11-25T03:37:24,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-25T03:37:24,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/0e4ae549bf65460ba3f50c694c4c2a93 is 50, key is test_row_0/C:col10/1732505843112/Put/seqid=0 2024-11-25T03:37:24,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741985_1161 (size=12001) 2024-11-25T03:37:24,851 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/0e4ae549bf65460ba3f50c694c4c2a93 2024-11-25T03:37:24,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/9d29a4d1d1f04166b8aec4f11a8ff5d8 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/9d29a4d1d1f04166b8aec4f11a8ff5d8 2024-11-25T03:37:24,866 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/9d29a4d1d1f04166b8aec4f11a8ff5d8, entries=150, sequenceid=95, filesize=30.2 K 2024-11-25T03:37:24,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/ab558a8ba57846bcad4896923a6392fc as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/ab558a8ba57846bcad4896923a6392fc 2024-11-25T03:37:24,872 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/ab558a8ba57846bcad4896923a6392fc, entries=150, sequenceid=95, filesize=11.7 K 2024-11-25T03:37:24,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/0e4ae549bf65460ba3f50c694c4c2a93 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/0e4ae549bf65460ba3f50c694c4c2a93 2024-11-25T03:37:24,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-25T03:37:24,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:24,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. as already flushing 2024-11-25T03:37:24,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:24,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:24,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:24,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:24,884 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/0e4ae549bf65460ba3f50c694c4c2a93, entries=150, sequenceid=95, filesize=11.7 K 2024-11-25T03:37:24,898 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 9e284372599441cd748eba8961521ba1 in 634ms, sequenceid=95, compaction requested=true 2024-11-25T03:37:24,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e284372599441cd748eba8961521ba1: 2024-11-25T03:37:24,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e284372599441cd748eba8961521ba1:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:37:24,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:24,898 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:37:24,898 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:37:24,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e284372599441cd748eba8961521ba1:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:37:24,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:24,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e284372599441cd748eba8961521ba1:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:37:24,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:37:24,900 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:37:24,900 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 9e284372599441cd748eba8961521ba1/B is initiating minor compaction (all files) 2024-11-25T03:37:24,900 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e284372599441cd748eba8961521ba1/B in TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:24,900 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/e71896520f18492fb2c734866bbb4e6c, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/2357accf81f04b9da6399c8a80f88e94, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/ab558a8ba57846bcad4896923a6392fc] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp, totalSize=35.3 K 2024-11-25T03:37:24,900 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:37:24,900 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): 9e284372599441cd748eba8961521ba1/A is initiating minor compaction (all files) 2024-11-25T03:37:24,900 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e284372599441cd748eba8961521ba1/A in TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:24,900 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/e9f2be7b55f94605a4e4bd7f4a76296e, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/353578fa21dc4a9cb549ae5e0108826f, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/9d29a4d1d1f04166b8aec4f11a8ff5d8] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp, totalSize=90.8 K 2024-11-25T03:37:24,900 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:24,900 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. files: [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/e9f2be7b55f94605a4e4bd7f4a76296e, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/353578fa21dc4a9cb549ae5e0108826f, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/9d29a4d1d1f04166b8aec4f11a8ff5d8] 2024-11-25T03:37:24,901 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting e71896520f18492fb2c734866bbb4e6c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732505841242 2024-11-25T03:37:24,901 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9f2be7b55f94605a4e4bd7f4a76296e, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732505841242 2024-11-25T03:37:24,902 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 2357accf81f04b9da6399c8a80f88e94, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732505841940 2024-11-25T03:37:24,902 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 353578fa21dc4a9cb549ae5e0108826f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732505841940 2024-11-25T03:37:24,902 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting ab558a8ba57846bcad4896923a6392fc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732505843112 2024-11-25T03:37:24,903 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d29a4d1d1f04166b8aec4f11a8ff5d8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732505843112 2024-11-25T03:37:24,916 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e284372599441cd748eba8961521ba1#B#compaction#140 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:37:24,917 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/7584232a4e154410b2e3fd1b6ca4bbf0 is 50, key is test_row_0/B:col10/1732505843112/Put/seqid=0 2024-11-25T03:37:24,925 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9e284372599441cd748eba8961521ba1] 2024-11-25T03:37:24,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 9e284372599441cd748eba8961521ba1 2024-11-25T03:37:24,934 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:37:24,934 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e284372599441cd748eba8961521ba1 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-25T03:37:24,945 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411259498adc3d9ca4907b967d5665d49099b_9e284372599441cd748eba8961521ba1 store=[table=TestAcidGuarantees family=A region=9e284372599441cd748eba8961521ba1] 2024-11-25T03:37:24,949 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411259498adc3d9ca4907b967d5665d49099b_9e284372599441cd748eba8961521ba1, store=[table=TestAcidGuarantees family=A region=9e284372599441cd748eba8961521ba1] 2024-11-25T03:37:24,950 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411259498adc3d9ca4907b967d5665d49099b_9e284372599441cd748eba8961521ba1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9e284372599441cd748eba8961521ba1] 2024-11-25T03:37:24,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741986_1162 (size=12207) 2024-11-25T03:37:24,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411250a63468da2be49868fe995815cc379a3_9e284372599441cd748eba8961521ba1 is 50, key is test_row_0/A:col10/1732505844932/Put/seqid=0 2024-11-25T03:37:24,977 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/7584232a4e154410b2e3fd1b6ca4bbf0 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/7584232a4e154410b2e3fd1b6ca4bbf0 2024-11-25T03:37:24,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741987_1163 (size=4469) 2024-11-25T03:37:24,997 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e284372599441cd748eba8961521ba1#A#compaction#141 average throughput is 0.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:37:24,998 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/cd605cec45e94ae089fb134d32ee4fc1 is 175, key is test_row_0/A:col10/1732505843112/Put/seqid=0 2024-11-25T03:37:24,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741988_1164 (size=12154) 2024-11-25T03:37:24,999 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:37:25,004 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411250a63468da2be49868fe995815cc379a3_9e284372599441cd748eba8961521ba1 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411250a63468da2be49868fe995815cc379a3_9e284372599441cd748eba8961521ba1 2024-11-25T03:37:25,005 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/0d3888a58d61480893e73268e3a745d2, store: [table=TestAcidGuarantees family=A region=9e284372599441cd748eba8961521ba1] 2024-11-25T03:37:25,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/0d3888a58d61480893e73268e3a745d2 is 175, key is test_row_0/A:col10/1732505844932/Put/seqid=0 2024-11-25T03:37:25,014 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e284372599441cd748eba8961521ba1/B of 9e284372599441cd748eba8961521ba1 into 7584232a4e154410b2e3fd1b6ca4bbf0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:37:25,014 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e284372599441cd748eba8961521ba1: 2024-11-25T03:37:25,014 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., storeName=9e284372599441cd748eba8961521ba1/B, priority=13, startTime=1732505844898; duration=0sec 2024-11-25T03:37:25,014 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:37:25,014 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e284372599441cd748eba8961521ba1:B 2024-11-25T03:37:25,014 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:37:25,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:25,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505904972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:25,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:25,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:25,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:25,017 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:37:25,017 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 9e284372599441cd748eba8961521ba1/C is initiating minor compaction (all files) 2024-11-25T03:37:25,017 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e284372599441cd748eba8961521ba1/C in TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:25,017 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/24f796b495bb410580e242aea444b720, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/aebda93c62b242a7a3fb8b403f59f0d0, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/0e4ae549bf65460ba3f50c694c4c2a93] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp, totalSize=35.3 K 2024-11-25T03:37:25,024 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 24f796b495bb410580e242aea444b720, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732505841242 2024-11-25T03:37:25,025 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting aebda93c62b242a7a3fb8b403f59f0d0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732505841940 2024-11-25T03:37:25,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:25,025 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e4ae549bf65460ba3f50c694c4c2a93, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732505843112 2024-11-25T03:37:25,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505905015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:25,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:25,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:25,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:25,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:25,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505905017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:25,029 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:25,029 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:25,029 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:25,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:25,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505905017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:25,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:25,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:25,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:25,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-25T03:37:25,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:25,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. as already flushing 2024-11-25T03:37:25,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:25,032 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:25,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:25,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:25,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-25T03:37:25,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741989_1165 (size=31161) 2024-11-25T03:37:25,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741990_1166 (size=30955) 2024-11-25T03:37:25,046 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/0d3888a58d61480893e73268e3a745d2 2024-11-25T03:37:25,055 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e284372599441cd748eba8961521ba1#C#compaction#143 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:37:25,056 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/726121ed23b44824ba99dae8b5d687f0 is 50, key is test_row_0/C:col10/1732505843112/Put/seqid=0 2024-11-25T03:37:25,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/686db1342ff94a9bbf27d3aec0877107 is 50, key is test_row_0/B:col10/1732505844932/Put/seqid=0 2024-11-25T03:37:25,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741991_1167 (size=12207) 2024-11-25T03:37:25,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741992_1168 (size=12001) 2024-11-25T03:37:25,112 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/686db1342ff94a9bbf27d3aec0877107 2024-11-25T03:37:25,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/de24325ce53249b197718ba311dcddae is 50, key is test_row_0/C:col10/1732505844932/Put/seqid=0 2024-11-25T03:37:25,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741993_1169 (size=12001) 2024-11-25T03:37:25,138 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/de24325ce53249b197718ba311dcddae 2024-11-25T03:37:25,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:25,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505905129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:25,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:25,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505905137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:25,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:25,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505905137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:25,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:25,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:25,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:25,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:25,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505905137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:25,141 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:25,141 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:25,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:25,141 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:25,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:25,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:25,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:25,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:25,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:25,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/0d3888a58d61480893e73268e3a745d2 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/0d3888a58d61480893e73268e3a745d2 2024-11-25T03:37:25,156 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/0d3888a58d61480893e73268e3a745d2, entries=150, sequenceid=116, filesize=30.2 K 2024-11-25T03:37:25,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/686db1342ff94a9bbf27d3aec0877107 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/686db1342ff94a9bbf27d3aec0877107 2024-11-25T03:37:25,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/686db1342ff94a9bbf27d3aec0877107, entries=150, sequenceid=116, filesize=11.7 K 2024-11-25T03:37:25,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/de24325ce53249b197718ba311dcddae as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/de24325ce53249b197718ba311dcddae 2024-11-25T03:37:25,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/de24325ce53249b197718ba311dcddae, entries=150, sequenceid=116, filesize=11.7 K 2024-11-25T03:37:25,182 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=93.93 KB/96180 for 9e284372599441cd748eba8961521ba1 in 248ms, sequenceid=116, compaction requested=false 2024-11-25T03:37:25,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e284372599441cd748eba8961521ba1: 2024-11-25T03:37:25,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-25T03:37:25,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:25,188 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 9e284372599441cd748eba8961521ba1 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-25T03:37:25,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125cf664b680e814ce7800349afda548565_9e284372599441cd748eba8961521ba1 is 50, key is test_row_0/A:col10/1732505844967/Put/seqid=0 2024-11-25T03:37:25,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741994_1170 (size=12204) 2024-11-25T03:37:25,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:37:25,237 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125cf664b680e814ce7800349afda548565_9e284372599441cd748eba8961521ba1 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125cf664b680e814ce7800349afda548565_9e284372599441cd748eba8961521ba1 2024-11-25T03:37:25,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/42743985edd94a6c9243bad2b63af770, store: [table=TestAcidGuarantees family=A region=9e284372599441cd748eba8961521ba1] 2024-11-25T03:37:25,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/42743985edd94a6c9243bad2b63af770 is 175, key is test_row_0/A:col10/1732505844967/Put/seqid=0 2024-11-25T03:37:25,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741995_1171 (size=31005) 2024-11-25T03:37:25,257 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/42743985edd94a6c9243bad2b63af770 2024-11-25T03:37:25,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/0e3f68dde8454979bdaff5d5cbda1924 is 50, key is test_row_0/B:col10/1732505844967/Put/seqid=0 2024-11-25T03:37:25,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741996_1172 (size=12051) 2024-11-25T03:37:25,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-25T03:37:25,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 9e284372599441cd748eba8961521ba1 2024-11-25T03:37:25,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. as already flushing 2024-11-25T03:37:25,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:25,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505905372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:25,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:25,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:25,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:25,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505905374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:25,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:25,384 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:25,384 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:25,384 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:25,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:25,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505905375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:25,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:25,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:25,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:25,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:25,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505905383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:25,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:25,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:25,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:25,452 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/cd605cec45e94ae089fb134d32ee4fc1 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/cd605cec45e94ae089fb134d32ee4fc1 2024-11-25T03:37:25,460 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e284372599441cd748eba8961521ba1/A of 9e284372599441cd748eba8961521ba1 into cd605cec45e94ae089fb134d32ee4fc1(size=30.4 K), total size for store is 60.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:37:25,460 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e284372599441cd748eba8961521ba1: 2024-11-25T03:37:25,460 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., storeName=9e284372599441cd748eba8961521ba1/A, priority=13, startTime=1732505844898; duration=0sec 2024-11-25T03:37:25,460 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:25,460 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e284372599441cd748eba8961521ba1:A 2024-11-25T03:37:25,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:25,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505905488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:25,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:25,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505905488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:25,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:25,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:25,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:25,496 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:25,496 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:25,496 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:25,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:25,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505905497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:25,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:25,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:25,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:25,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505905497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:25,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:25,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:25,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:25,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:25,513 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/726121ed23b44824ba99dae8b5d687f0 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/726121ed23b44824ba99dae8b5d687f0 2024-11-25T03:37:25,522 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e284372599441cd748eba8961521ba1/C of 9e284372599441cd748eba8961521ba1 into 726121ed23b44824ba99dae8b5d687f0(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:37:25,522 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e284372599441cd748eba8961521ba1: 2024-11-25T03:37:25,522 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., storeName=9e284372599441cd748eba8961521ba1/C, priority=13, startTime=1732505844898; duration=0sec 2024-11-25T03:37:25,522 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:25,522 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e284372599441cd748eba8961521ba1:C 2024-11-25T03:37:25,684 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/0e3f68dde8454979bdaff5d5cbda1924 2024-11-25T03:37:25,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/2d9b58e0c5dc4a699c6df7a471d6742d is 50, key is test_row_0/C:col10/1732505844967/Put/seqid=0 2024-11-25T03:37:25,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741997_1173 (size=12051) 2024-11-25T03:37:25,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:25,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505905697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:25,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:25,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:25,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:25,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:25,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505905708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:25,712 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:25,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:25,712 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:25,712 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:25,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505905708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:25,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:25,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:25,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:25,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:25,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505905708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:25,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:25,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:25,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:25,798 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T03:37:25,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-25T03:37:26,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:26,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505906008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:26,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:26,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:26,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:26,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:26,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505906017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:26,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:26,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505906017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:26,021 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:26,021 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:26,021 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:26,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:26,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505906017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:26,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:26,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:26,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:26,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:26,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:26,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:26,100 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/2d9b58e0c5dc4a699c6df7a471d6742d 2024-11-25T03:37:26,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/42743985edd94a6c9243bad2b63af770 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/42743985edd94a6c9243bad2b63af770 2024-11-25T03:37:26,115 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/42743985edd94a6c9243bad2b63af770, entries=150, sequenceid=134, filesize=30.3 K 2024-11-25T03:37:26,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/0e3f68dde8454979bdaff5d5cbda1924 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/0e3f68dde8454979bdaff5d5cbda1924 2024-11-25T03:37:26,124 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/0e3f68dde8454979bdaff5d5cbda1924, entries=150, sequenceid=134, filesize=11.8 K 2024-11-25T03:37:26,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/2d9b58e0c5dc4a699c6df7a471d6742d as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/2d9b58e0c5dc4a699c6df7a471d6742d 2024-11-25T03:37:26,139 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/2d9b58e0c5dc4a699c6df7a471d6742d, entries=150, sequenceid=134, filesize=11.8 K 2024-11-25T03:37:26,140 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 9e284372599441cd748eba8961521ba1 in 952ms, sequenceid=134, compaction requested=true 2024-11-25T03:37:26,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 9e284372599441cd748eba8961521ba1: 2024-11-25T03:37:26,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:26,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-25T03:37:26,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-25T03:37:26,144 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-25T03:37:26,144 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4200 sec 2024-11-25T03:37:26,146 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.4260 sec 2024-11-25T03:37:26,247 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-25T03:37:26,247 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-25T03:37:26,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 9e284372599441cd748eba8961521ba1 2024-11-25T03:37:26,522 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:37:26,522 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e284372599441cd748eba8961521ba1 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-25T03:37:26,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411252f97ae056d144344a371f51aa95ba6d1_9e284372599441cd748eba8961521ba1 is 50, key is test_row_0/A:col10/1732505846520/Put/seqid=0 2024-11-25T03:37:26,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741998_1174 (size=14794) 2024-11-25T03:37:26,547 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:37:26,553 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411252f97ae056d144344a371f51aa95ba6d1_9e284372599441cd748eba8961521ba1 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411252f97ae056d144344a371f51aa95ba6d1_9e284372599441cd748eba8961521ba1 2024-11-25T03:37:26,555 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/469bb20562984f43a0f460f212ae86f3, store: [table=TestAcidGuarantees family=A region=9e284372599441cd748eba8961521ba1] 2024-11-25T03:37:26,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/469bb20562984f43a0f460f212ae86f3 is 175, key is test_row_0/A:col10/1732505846520/Put/seqid=0 2024-11-25T03:37:26,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:26,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:26,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505906561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:26,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505906561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:26,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:26,568 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:26,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:26,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505906561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:26,568 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:26,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:26,568 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:26,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:26,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:26,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:26,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:26,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:26,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505906568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:26,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:26,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:26,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:26,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741999_1175 (size=39749) 2024-11-25T03:37:26,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:26,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505906678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:26,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:26,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505906678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:26,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:26,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:26,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505906678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:26,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:26,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:26,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:26,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:26,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:26,683 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:26,683 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:26,683 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:26,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:26,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505906684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:26,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:26,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:26,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:26,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-25T03:37:26,867 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:37:26,868 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:37:26,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-25T03:37:26,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-25T03:37:26,871 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:37:26,871 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:37:26,872 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:37:26,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:26,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505906887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:26,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:26,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:26,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:26,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:26,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:26,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505906887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:26,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505906887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:26,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:26,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505906888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:26,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:26,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:26,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:26,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:26,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:26,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:26,894 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:26,894 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:26,894 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:26,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-25T03:37:27,008 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/469bb20562984f43a0f460f212ae86f3 2024-11-25T03:37:27,017 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/a39599753f644afe99f11efa81b0bda7 is 50, key is test_row_0/B:col10/1732505846520/Put/seqid=0 2024-11-25T03:37:27,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742000_1176 (size=12151) 2024-11-25T03:37:27,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-25T03:37:27,024 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/a39599753f644afe99f11efa81b0bda7 2024-11-25T03:37:27,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:27,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. as already flushing 2024-11-25T03:37:27,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:27,024 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:27,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:27,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:27,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/6dab53fa845a4d489cce677f6b07425a is 50, key is test_row_0/C:col10/1732505846520/Put/seqid=0 2024-11-25T03:37:27,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742001_1177 (size=12151) 2024-11-25T03:37:27,056 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/6dab53fa845a4d489cce677f6b07425a 2024-11-25T03:37:27,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/469bb20562984f43a0f460f212ae86f3 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/469bb20562984f43a0f460f212ae86f3 2024-11-25T03:37:27,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/469bb20562984f43a0f460f212ae86f3, entries=200, sequenceid=156, filesize=38.8 K 2024-11-25T03:37:27,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/a39599753f644afe99f11efa81b0bda7 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/a39599753f644afe99f11efa81b0bda7 2024-11-25T03:37:27,078 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/a39599753f644afe99f11efa81b0bda7, entries=150, sequenceid=156, filesize=11.9 K 2024-11-25T03:37:27,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/6dab53fa845a4d489cce677f6b07425a as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/6dab53fa845a4d489cce677f6b07425a 2024-11-25T03:37:27,087 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/6dab53fa845a4d489cce677f6b07425a, entries=150, sequenceid=156, filesize=11.9 K 2024-11-25T03:37:27,088 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 9e284372599441cd748eba8961521ba1 in 566ms, sequenceid=156, compaction requested=true 2024-11-25T03:37:27,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e284372599441cd748eba8961521ba1: 2024-11-25T03:37:27,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e284372599441cd748eba8961521ba1:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:37:27,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:27,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e284372599441cd748eba8961521ba1:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:37:27,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:37:27,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e284372599441cd748eba8961521ba1:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:37:27,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T03:37:27,089 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:37:27,090 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:37:27,091 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:37:27,091 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 9e284372599441cd748eba8961521ba1/B is initiating minor compaction (all files) 2024-11-25T03:37:27,091 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e284372599441cd748eba8961521ba1/B in TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:27,091 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/7584232a4e154410b2e3fd1b6ca4bbf0, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/686db1342ff94a9bbf27d3aec0877107, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/0e3f68dde8454979bdaff5d5cbda1924, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/a39599753f644afe99f11efa81b0bda7] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp, totalSize=47.3 K 2024-11-25T03:37:27,092 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132870 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:37:27,092 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 7584232a4e154410b2e3fd1b6ca4bbf0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732505843112 2024-11-25T03:37:27,092 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): 9e284372599441cd748eba8961521ba1/A is initiating minor compaction (all files) 2024-11-25T03:37:27,092 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e284372599441cd748eba8961521ba1/A in TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:27,092 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/cd605cec45e94ae089fb134d32ee4fc1, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/0d3888a58d61480893e73268e3a745d2, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/42743985edd94a6c9243bad2b63af770, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/469bb20562984f43a0f460f212ae86f3] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp, totalSize=129.8 K 2024-11-25T03:37:27,092 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:27,093 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. files: [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/cd605cec45e94ae089fb134d32ee4fc1, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/0d3888a58d61480893e73268e3a745d2, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/42743985edd94a6c9243bad2b63af770, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/469bb20562984f43a0f460f212ae86f3] 2024-11-25T03:37:27,093 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 686db1342ff94a9bbf27d3aec0877107, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732505844299 2024-11-25T03:37:27,093 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd605cec45e94ae089fb134d32ee4fc1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732505843112 2024-11-25T03:37:27,093 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e3f68dde8454979bdaff5d5cbda1924, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732505844964 2024-11-25T03:37:27,093 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting a39599753f644afe99f11efa81b0bda7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732505845373 2024-11-25T03:37:27,093 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d3888a58d61480893e73268e3a745d2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732505844299 2024-11-25T03:37:27,094 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42743985edd94a6c9243bad2b63af770, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732505844964 2024-11-25T03:37:27,095 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 469bb20562984f43a0f460f212ae86f3, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732505845369 2024-11-25T03:37:27,108 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e284372599441cd748eba8961521ba1#B#compaction#152 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:37:27,109 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/dd60528f421b4b6a981ad60ee64f5189 is 50, key is test_row_0/B:col10/1732505846520/Put/seqid=0 2024-11-25T03:37:27,110 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9e284372599441cd748eba8961521ba1] 2024-11-25T03:37:27,123 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411254856658c4e514d92a0f574a38a1a418b_9e284372599441cd748eba8961521ba1 store=[table=TestAcidGuarantees family=A region=9e284372599441cd748eba8961521ba1] 2024-11-25T03:37:27,126 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411254856658c4e514d92a0f574a38a1a418b_9e284372599441cd748eba8961521ba1, store=[table=TestAcidGuarantees family=A region=9e284372599441cd748eba8961521ba1] 2024-11-25T03:37:27,126 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411254856658c4e514d92a0f574a38a1a418b_9e284372599441cd748eba8961521ba1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9e284372599441cd748eba8961521ba1] 2024-11-25T03:37:27,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742002_1178 (size=12493) 2024-11-25T03:37:27,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742003_1179 (size=4469) 2024-11-25T03:37:27,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-25T03:37:27,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:27,178 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 9e284372599441cd748eba8961521ba1 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-25T03:37:27,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-25T03:37:27,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112518145d7828a04a25be2fad49e076c89d_9e284372599441cd748eba8961521ba1 is 50, key is test_row_0/A:col10/1732505846553/Put/seqid=0 2024-11-25T03:37:27,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742004_1180 (size=12304) 2024-11-25T03:37:27,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 9e284372599441cd748eba8961521ba1 2024-11-25T03:37:27,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. as already flushing 2024-11-25T03:37:27,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:27,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:27,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505907224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:27,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505907225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:27,231 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:27,231 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:27,231 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:27,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:27,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:27,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:27,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:27,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505907231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:27,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:27,236 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:27,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505907231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:27,236 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:27,236 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:27,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:27,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:27,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:27,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:27,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505907337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:27,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:27,341 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:27,341 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:27,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505907337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:27,341 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:27,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:27,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:27,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:27,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:27,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505907347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:27,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:27,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:27,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505907347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:27,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:27,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:27,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:27,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:27,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:27,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-25T03:37:27,546 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/dd60528f421b4b6a981ad60ee64f5189 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/dd60528f421b4b6a981ad60ee64f5189 2024-11-25T03:37:27,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:27,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505907547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:27,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:27,552 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:27,552 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:27,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505907547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:27,552 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:27,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:27,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:27,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:27,553 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9e284372599441cd748eba8961521ba1/B of 9e284372599441cd748eba8961521ba1 into dd60528f421b4b6a981ad60ee64f5189(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:37:27,553 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e284372599441cd748eba8961521ba1: 2024-11-25T03:37:27,553 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., storeName=9e284372599441cd748eba8961521ba1/B, priority=12, startTime=1732505847089; duration=0sec 2024-11-25T03:37:27,553 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:37:27,553 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e284372599441cd748eba8961521ba1:B 2024-11-25T03:37:27,553 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:37:27,555 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:37:27,555 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 9e284372599441cd748eba8961521ba1/C is initiating minor compaction (all files) 2024-11-25T03:37:27,555 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e284372599441cd748eba8961521ba1/C in TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:27,555 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/726121ed23b44824ba99dae8b5d687f0, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/de24325ce53249b197718ba311dcddae, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/2d9b58e0c5dc4a699c6df7a471d6742d, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/6dab53fa845a4d489cce677f6b07425a] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp, totalSize=47.3 K 2024-11-25T03:37:27,556 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 726121ed23b44824ba99dae8b5d687f0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732505843112 2024-11-25T03:37:27,556 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting de24325ce53249b197718ba311dcddae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732505844299 2024-11-25T03:37:27,557 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d9b58e0c5dc4a699c6df7a471d6742d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732505844964 2024-11-25T03:37:27,557 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 6dab53fa845a4d489cce677f6b07425a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732505845373 2024-11-25T03:37:27,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:27,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505907558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:27,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:27,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:27,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:27,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:27,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505907558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:27,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:27,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:27,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:27,566 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e284372599441cd748eba8961521ba1#C#compaction#155 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:37:27,567 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e284372599441cd748eba8961521ba1#A#compaction#153 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:37:27,567 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/5333c98762ac477598ca290db53bc2b9 is 50, key is test_row_0/C:col10/1732505846520/Put/seqid=0 2024-11-25T03:37:27,568 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/f05491e09c4c4c1283029d4b9f8e8943 is 175, key is test_row_0/A:col10/1732505846520/Put/seqid=0 2024-11-25T03:37:27,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742005_1181 (size=12493) 2024-11-25T03:37:27,579 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/5333c98762ac477598ca290db53bc2b9 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/5333c98762ac477598ca290db53bc2b9 2024-11-25T03:37:27,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742006_1182 (size=31447) 2024-11-25T03:37:27,588 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/f05491e09c4c4c1283029d4b9f8e8943 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/f05491e09c4c4c1283029d4b9f8e8943 2024-11-25T03:37:27,590 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9e284372599441cd748eba8961521ba1/C of 9e284372599441cd748eba8961521ba1 into 5333c98762ac477598ca290db53bc2b9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:37:27,591 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e284372599441cd748eba8961521ba1: 2024-11-25T03:37:27,591 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., storeName=9e284372599441cd748eba8961521ba1/C, priority=12, startTime=1732505847089; duration=0sec 2024-11-25T03:37:27,591 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:27,591 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e284372599441cd748eba8961521ba1:C 2024-11-25T03:37:27,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:37:27,595 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9e284372599441cd748eba8961521ba1/A of 9e284372599441cd748eba8961521ba1 into f05491e09c4c4c1283029d4b9f8e8943(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:37:27,595 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e284372599441cd748eba8961521ba1: 2024-11-25T03:37:27,595 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., storeName=9e284372599441cd748eba8961521ba1/A, priority=12, startTime=1732505847089; duration=0sec 2024-11-25T03:37:27,595 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:27,595 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e284372599441cd748eba8961521ba1:A 2024-11-25T03:37:27,598 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112518145d7828a04a25be2fad49e076c89d_9e284372599441cd748eba8961521ba1 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112518145d7828a04a25be2fad49e076c89d_9e284372599441cd748eba8961521ba1 2024-11-25T03:37:27,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/f79146d967b6470a975c6940ecb1b894, store: [table=TestAcidGuarantees family=A region=9e284372599441cd748eba8961521ba1] 2024-11-25T03:37:27,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/f79146d967b6470a975c6940ecb1b894 is 175, key is test_row_0/A:col10/1732505846553/Put/seqid=0 2024-11-25T03:37:27,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742007_1183 (size=31105) 2024-11-25T03:37:27,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:27,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505907857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:27,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:27,860 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:27,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505907857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:27,860 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:27,860 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:27,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:27,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:27,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:27,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:27,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505907867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:27,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:27,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:27,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505907867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:27,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:27,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:27,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:27,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:27,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:28,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-25T03:37:28,015 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/f79146d967b6470a975c6940ecb1b894 2024-11-25T03:37:28,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/527decd5ff1f4174b80817cf5f2df565 is 50, key is test_row_0/B:col10/1732505846553/Put/seqid=0 2024-11-25T03:37:28,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742008_1184 (size=12151) 2024-11-25T03:37:28,031 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/527decd5ff1f4174b80817cf5f2df565 2024-11-25T03:37:28,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/0b6bbb30e28d433cb212b4ab000205ef is 50, key is test_row_0/C:col10/1732505846553/Put/seqid=0 2024-11-25T03:37:28,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742009_1185 (size=12151) 2024-11-25T03:37:28,045 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/0b6bbb30e28d433cb212b4ab000205ef 2024-11-25T03:37:28,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/f79146d967b6470a975c6940ecb1b894 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/f79146d967b6470a975c6940ecb1b894 2024-11-25T03:37:28,055 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/f79146d967b6470a975c6940ecb1b894, entries=150, sequenceid=172, filesize=30.4 K 2024-11-25T03:37:28,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/527decd5ff1f4174b80817cf5f2df565 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/527decd5ff1f4174b80817cf5f2df565 2024-11-25T03:37:28,060 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/527decd5ff1f4174b80817cf5f2df565, entries=150, sequenceid=172, filesize=11.9 K 2024-11-25T03:37:28,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/0b6bbb30e28d433cb212b4ab000205ef as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/0b6bbb30e28d433cb212b4ab000205ef 2024-11-25T03:37:28,065 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/0b6bbb30e28d433cb212b4ab000205ef, entries=150, sequenceid=172, filesize=11.9 K 2024-11-25T03:37:28,066 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 9e284372599441cd748eba8961521ba1 in 888ms, sequenceid=172, compaction requested=false 2024-11-25T03:37:28,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 9e284372599441cd748eba8961521ba1: 2024-11-25T03:37:28,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:28,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-25T03:37:28,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-25T03:37:28,070 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-25T03:37:28,070 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1960 sec 2024-11-25T03:37:28,072 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.2030 sec 2024-11-25T03:37:28,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 9e284372599441cd748eba8961521ba1 2024-11-25T03:37:28,371 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:37:28,371 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e284372599441cd748eba8961521ba1 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-25T03:37:28,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125c467437032794a2f8863def8d8eb1ef2_9e284372599441cd748eba8961521ba1 is 50, key is test_row_0/A:col10/1732505848371/Put/seqid=0 2024-11-25T03:37:28,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742010_1186 (size=14794) 2024-11-25T03:37:28,386 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:37:28,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:28,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505908385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:28,390 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125c467437032794a2f8863def8d8eb1ef2_9e284372599441cd748eba8961521ba1 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125c467437032794a2f8863def8d8eb1ef2_9e284372599441cd748eba8961521ba1 2024-11-25T03:37:28,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:28,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:28,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:28,391 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/8dc28716bc184d408476d21409d399a6, store: [table=TestAcidGuarantees family=A region=9e284372599441cd748eba8961521ba1] 2024-11-25T03:37:28,392 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/8dc28716bc184d408476d21409d399a6 is 175, key is test_row_0/A:col10/1732505848371/Put/seqid=0 2024-11-25T03:37:28,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:28,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505908388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:28,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:28,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505908389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:28,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:28,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:28,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:28,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:28,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505908390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:28,395 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:28,395 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:28,395 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:28,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:28,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:28,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:28,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742011_1187 (size=39749) 2024-11-25T03:37:28,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:28,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505908497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:28,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:28,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505908497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:28,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:28,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:28,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:28,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:28,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:28,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505908497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:28,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:28,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:28,499 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:28,499 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:28,499 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:28,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:28,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505908507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:28,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:28,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:28,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:28,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:28,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505908708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:28,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:28,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:28,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505908708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:28,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:28,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:28,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:28,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505908708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:28,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:28,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:28,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:28,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:28,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:28,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:28,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:28,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505908717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:28,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:28,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:28,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:28,799 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=197, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/8dc28716bc184d408476d21409d399a6 2024-11-25T03:37:28,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/2fa3fade0341434c953910057e73f310 is 50, key is test_row_0/B:col10/1732505848371/Put/seqid=0 2024-11-25T03:37:28,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742012_1188 (size=12151) 2024-11-25T03:37:28,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:28,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1732505908827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:28,831 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,34881,1732505806588 for 'test_row_2' in 9e284372599441cd748eba8961521ba1 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8257 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:37:28,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:28,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:28,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:29,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:29,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505909017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:29,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:29,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:29,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505909017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:29,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:29,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:29,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:29,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:29,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:29,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-25T03:37:29,028 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:37:29,028 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:37:29,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-25T03:37:29,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:29,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505909028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:29,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-25T03:37:29,031 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:37:29,031 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:29,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:29,031 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:29,031 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:29,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505909028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:29,031 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:37:29,031 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:37:29,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:29,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:29,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:29,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-25T03:37:29,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-25T03:37:29,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:29,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. as already flushing 2024-11-25T03:37:29,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:29,184 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:29,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:29,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:29,212 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/2fa3fade0341434c953910057e73f310 2024-11-25T03:37:29,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/cfcf0bba89f14c3b979e3115b0b1b629 is 50, key is test_row_0/C:col10/1732505848371/Put/seqid=0 2024-11-25T03:37:29,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742013_1189 (size=12151) 2024-11-25T03:37:29,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-25T03:37:29,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:29,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. as already flushing 2024-11-25T03:37:29,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:29,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:29,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:29,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:29,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-25T03:37:29,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-25T03:37:29,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:29,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. as already flushing 2024-11-25T03:37:29,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:29,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:29,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:29,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:29,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:29,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505909528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:29,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:29,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:29,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505909528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:29,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:29,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:29,530 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:29,530 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:29,530 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:29,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:29,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505909537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:29,542 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:29,542 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:29,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:29,542 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:29,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505909537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:29,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:29,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:29,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:29,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/cfcf0bba89f14c3b979e3115b0b1b629 2024-11-25T03:37:29,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/8dc28716bc184d408476d21409d399a6 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/8dc28716bc184d408476d21409d399a6 2024-11-25T03:37:29,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/8dc28716bc184d408476d21409d399a6, entries=200, sequenceid=197, filesize=38.8 K 2024-11-25T03:37:29,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/2fa3fade0341434c953910057e73f310 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/2fa3fade0341434c953910057e73f310 2024-11-25T03:37:29,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-25T03:37:29,643 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/2fa3fade0341434c953910057e73f310, entries=150, sequenceid=197, filesize=11.9 K 2024-11-25T03:37:29,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:29,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. as already flushing 2024-11-25T03:37:29,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:29,644 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:29,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:29,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/cfcf0bba89f14c3b979e3115b0b1b629 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/cfcf0bba89f14c3b979e3115b0b1b629 2024-11-25T03:37:29,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:29,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/cfcf0bba89f14c3b979e3115b0b1b629, entries=150, sequenceid=197, filesize=11.9 K 2024-11-25T03:37:29,650 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 9e284372599441cd748eba8961521ba1 in 1279ms, sequenceid=197, compaction requested=true 2024-11-25T03:37:29,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e284372599441cd748eba8961521ba1: 2024-11-25T03:37:29,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e284372599441cd748eba8961521ba1:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:37:29,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:29,650 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:37:29,650 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:37:29,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e284372599441cd748eba8961521ba1:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:37:29,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:29,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e284372599441cd748eba8961521ba1:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:37:29,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:37:29,651 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:37:29,651 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): 9e284372599441cd748eba8961521ba1/A is initiating minor compaction (all files) 2024-11-25T03:37:29,651 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e284372599441cd748eba8961521ba1/A in TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:29,652 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:37:29,652 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 9e284372599441cd748eba8961521ba1/B is initiating minor compaction (all files) 2024-11-25T03:37:29,652 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/f05491e09c4c4c1283029d4b9f8e8943, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/f79146d967b6470a975c6940ecb1b894, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/8dc28716bc184d408476d21409d399a6] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp, totalSize=99.9 K 2024-11-25T03:37:29,652 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e284372599441cd748eba8961521ba1/B in TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:29,652 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:29,652 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. files: [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/f05491e09c4c4c1283029d4b9f8e8943, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/f79146d967b6470a975c6940ecb1b894, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/8dc28716bc184d408476d21409d399a6] 2024-11-25T03:37:29,652 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/dd60528f421b4b6a981ad60ee64f5189, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/527decd5ff1f4174b80817cf5f2df565, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/2fa3fade0341434c953910057e73f310] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp, totalSize=35.9 K 2024-11-25T03:37:29,652 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting dd60528f421b4b6a981ad60ee64f5189, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732505845373 2024-11-25T03:37:29,652 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting f05491e09c4c4c1283029d4b9f8e8943, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732505845373 2024-11-25T03:37:29,653 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 527decd5ff1f4174b80817cf5f2df565, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732505846553 2024-11-25T03:37:29,653 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting f79146d967b6470a975c6940ecb1b894, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732505846553 2024-11-25T03:37:29,653 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fa3fade0341434c953910057e73f310, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732505847229 2024-11-25T03:37:29,653 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8dc28716bc184d408476d21409d399a6, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732505847224 2024-11-25T03:37:29,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-25T03:37:29,660 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9e284372599441cd748eba8961521ba1] 2024-11-25T03:37:29,661 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e284372599441cd748eba8961521ba1#B#compaction#161 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:37:29,662 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/5cbf16466bb84c0296892deebc5530e5 is 50, key is test_row_0/B:col10/1732505848371/Put/seqid=0 2024-11-25T03:37:29,662 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125bdf30ee8fe4f4dd584171d6ee701d937_9e284372599441cd748eba8961521ba1 store=[table=TestAcidGuarantees family=A region=9e284372599441cd748eba8961521ba1] 2024-11-25T03:37:29,665 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125bdf30ee8fe4f4dd584171d6ee701d937_9e284372599441cd748eba8961521ba1, store=[table=TestAcidGuarantees family=A region=9e284372599441cd748eba8961521ba1] 2024-11-25T03:37:29,666 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125bdf30ee8fe4f4dd584171d6ee701d937_9e284372599441cd748eba8961521ba1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9e284372599441cd748eba8961521ba1] 2024-11-25T03:37:29,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742014_1190 (size=12595) 2024-11-25T03:37:29,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742015_1191 (size=4469) 2024-11-25T03:37:29,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-25T03:37:29,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:29,797 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 9e284372599441cd748eba8961521ba1 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-25T03:37:29,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125de3b812063d04d3ea8a25ac82985eabe_9e284372599441cd748eba8961521ba1 is 50, key is test_row_0/A:col10/1732505848382/Put/seqid=0 2024-11-25T03:37:29,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742016_1192 (size=12304) 2024-11-25T03:37:30,074 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/5cbf16466bb84c0296892deebc5530e5 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/5cbf16466bb84c0296892deebc5530e5 2024-11-25T03:37:30,079 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e284372599441cd748eba8961521ba1#A#compaction#162 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:37:30,079 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e284372599441cd748eba8961521ba1/B of 9e284372599441cd748eba8961521ba1 into 5cbf16466bb84c0296892deebc5530e5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:37:30,079 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e284372599441cd748eba8961521ba1: 2024-11-25T03:37:30,079 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., storeName=9e284372599441cd748eba8961521ba1/B, priority=13, startTime=1732505849650; duration=0sec 2024-11-25T03:37:30,080 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:37:30,080 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e284372599441cd748eba8961521ba1:B 2024-11-25T03:37:30,080 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/72b08dfa6931432aa28e67ca5ff2927b is 175, key is test_row_0/A:col10/1732505848371/Put/seqid=0 2024-11-25T03:37:30,080 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:37:30,081 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:37:30,081 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 9e284372599441cd748eba8961521ba1/C is initiating minor compaction (all files) 2024-11-25T03:37:30,081 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e284372599441cd748eba8961521ba1/C in TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:30,081 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/5333c98762ac477598ca290db53bc2b9, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/0b6bbb30e28d433cb212b4ab000205ef, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/cfcf0bba89f14c3b979e3115b0b1b629] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp, totalSize=35.9 K 2024-11-25T03:37:30,082 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 5333c98762ac477598ca290db53bc2b9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732505845373 2024-11-25T03:37:30,082 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b6bbb30e28d433cb212b4ab000205ef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732505846553 2024-11-25T03:37:30,083 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting cfcf0bba89f14c3b979e3115b0b1b629, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732505847229 2024-11-25T03:37:30,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742017_1193 (size=31549) 2024-11-25T03:37:30,094 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e284372599441cd748eba8961521ba1#C#compaction#164 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:37:30,094 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/e1327af6e0644ae09c114c28c5ecc1d2 is 50, key is test_row_0/C:col10/1732505848371/Put/seqid=0 2024-11-25T03:37:30,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742018_1194 (size=12595) 2024-11-25T03:37:30,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-25T03:37:30,191 INFO [master/38a5d2077268:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-25T03:37:30,191 INFO [master/38a5d2077268:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-25T03:37:30,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:37:30,217 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125de3b812063d04d3ea8a25ac82985eabe_9e284372599441cd748eba8961521ba1 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125de3b812063d04d3ea8a25ac82985eabe_9e284372599441cd748eba8961521ba1 2024-11-25T03:37:30,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/25185d89ded74a6abfe8d4af149cce41, store: [table=TestAcidGuarantees family=A region=9e284372599441cd748eba8961521ba1] 2024-11-25T03:37:30,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/25185d89ded74a6abfe8d4af149cce41 is 175, key is test_row_0/A:col10/1732505848382/Put/seqid=0 2024-11-25T03:37:30,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742019_1195 (size=31105) 2024-11-25T03:37:30,231 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/25185d89ded74a6abfe8d4af149cce41 2024-11-25T03:37:30,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/2c19961d7bd3438d96bd08f57b64f101 is 50, key is test_row_0/B:col10/1732505848382/Put/seqid=0 2024-11-25T03:37:30,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742020_1196 (size=12151) 2024-11-25T03:37:30,494 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/72b08dfa6931432aa28e67ca5ff2927b as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/72b08dfa6931432aa28e67ca5ff2927b 2024-11-25T03:37:30,500 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e284372599441cd748eba8961521ba1/A of 9e284372599441cd748eba8961521ba1 into 72b08dfa6931432aa28e67ca5ff2927b(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:37:30,500 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e284372599441cd748eba8961521ba1: 2024-11-25T03:37:30,500 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., storeName=9e284372599441cd748eba8961521ba1/A, priority=13, startTime=1732505849650; duration=0sec 2024-11-25T03:37:30,501 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:30,501 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e284372599441cd748eba8961521ba1:A 2024-11-25T03:37:30,508 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/e1327af6e0644ae09c114c28c5ecc1d2 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/e1327af6e0644ae09c114c28c5ecc1d2 2024-11-25T03:37:30,514 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e284372599441cd748eba8961521ba1/C of 9e284372599441cd748eba8961521ba1 into e1327af6e0644ae09c114c28c5ecc1d2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:37:30,514 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e284372599441cd748eba8961521ba1: 2024-11-25T03:37:30,514 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., storeName=9e284372599441cd748eba8961521ba1/C, priority=13, startTime=1732505849651; duration=0sec 2024-11-25T03:37:30,514 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:30,514 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e284372599441cd748eba8961521ba1:C 2024-11-25T03:37:30,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 9e284372599441cd748eba8961521ba1 2024-11-25T03:37:30,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. as already flushing 2024-11-25T03:37:30,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:30,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505910568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:30,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:30,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:30,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:30,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:30,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505910607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:30,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:30,615 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:30,615 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:30,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505910607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:30,615 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:30,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:30,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:30,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:30,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:30,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505910607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:30,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:30,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:30,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:30,660 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/2c19961d7bd3438d96bd08f57b64f101 2024-11-25T03:37:30,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/c105947ef05a4ab3b1731c6d2e9faa7d is 50, key is test_row_0/C:col10/1732505848382/Put/seqid=0 2024-11-25T03:37:30,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742021_1197 (size=12151) 2024-11-25T03:37:30,674 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/c105947ef05a4ab3b1731c6d2e9faa7d 2024-11-25T03:37:30,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/25185d89ded74a6abfe8d4af149cce41 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/25185d89ded74a6abfe8d4af149cce41 2024-11-25T03:37:30,684 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/25185d89ded74a6abfe8d4af149cce41, entries=150, sequenceid=211, filesize=30.4 K 2024-11-25T03:37:30,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/2c19961d7bd3438d96bd08f57b64f101 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/2c19961d7bd3438d96bd08f57b64f101 2024-11-25T03:37:30,689 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/2c19961d7bd3438d96bd08f57b64f101, entries=150, sequenceid=211, filesize=11.9 K 2024-11-25T03:37:30,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/c105947ef05a4ab3b1731c6d2e9faa7d as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/c105947ef05a4ab3b1731c6d2e9faa7d 2024-11-25T03:37:30,695 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/c105947ef05a4ab3b1731c6d2e9faa7d, entries=150, sequenceid=211, filesize=11.9 K 2024-11-25T03:37:30,697 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=140.89 KB/144270 for 9e284372599441cd748eba8961521ba1 in 900ms, sequenceid=211, compaction requested=false 2024-11-25T03:37:30,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 9e284372599441cd748eba8961521ba1: 2024-11-25T03:37:30,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:30,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-25T03:37:30,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-25T03:37:30,700 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-25T03:37:30,700 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6670 sec 2024-11-25T03:37:30,703 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.6720 sec 2024-11-25T03:37:30,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 9e284372599441cd748eba8961521ba1 2024-11-25T03:37:30,724 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:37:30,724 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e284372599441cd748eba8961521ba1 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-25T03:37:30,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125d50fc6308edd4f60bf2cf3f7f5e879c5_9e284372599441cd748eba8961521ba1 is 50, key is test_row_0/A:col10/1732505850723/Put/seqid=0 2024-11-25T03:37:30,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742022_1198 (size=12304) 2024-11-25T03:37:30,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:30,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505910736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:30,776 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:30,776 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:30,776 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:30,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:30,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505910776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:30,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:30,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:30,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:30,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:30,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505910776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:30,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:30,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505910776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:30,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:30,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:30,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:30,782 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:30,782 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:30,782 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:30,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:30,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505910887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:30,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:30,890 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:30,890 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:30,890 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:30,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505910887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:30,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:30,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:30,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:30,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:30,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505910888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:30,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:30,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505910888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:30,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:30,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:30,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:30,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:30,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:30,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:31,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:31,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505911097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:31,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:31,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505911097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:31,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:31,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:31,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:31,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:31,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505911097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:31,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:31,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505911098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:31,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:31,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:31,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:31,104 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:31,104 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:31,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:31,104 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:31,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:31,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:31,156 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:37:31,161 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125d50fc6308edd4f60bf2cf3f7f5e879c5_9e284372599441cd748eba8961521ba1 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125d50fc6308edd4f60bf2cf3f7f5e879c5_9e284372599441cd748eba8961521ba1 2024-11-25T03:37:31,165 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/8431d0af668b4289927731f22e0ad62e, store: [table=TestAcidGuarantees family=A region=9e284372599441cd748eba8961521ba1] 2024-11-25T03:37:31,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/8431d0af668b4289927731f22e0ad62e is 175, key is test_row_0/A:col10/1732505850723/Put/seqid=0 2024-11-25T03:37:31,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-25T03:37:31,178 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:37:31,178 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:37:31,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-25T03:37:31,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-25T03:37:31,183 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:37:31,184 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:37:31,184 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:37:31,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742023_1199 (size=31105) 2024-11-25T03:37:31,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-25T03:37:31,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-25T03:37:31,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:31,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. as already flushing 2024-11-25T03:37:31,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:31,338 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:31,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:31,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:31,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:31,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505911407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:31,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:31,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505911408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:31,411 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:31,411 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:31,411 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:31,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:31,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:31,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:31,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:31,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505911408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:31,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:31,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:31,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:31,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:31,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505911409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:31,414 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:31,414 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:31,414 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:31,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-25T03:37:31,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:31,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. as already flushing 2024-11-25T03:37:31,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:31,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:31,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:31,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:31,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-25T03:37:31,587 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=241, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/8431d0af668b4289927731f22e0ad62e 2024-11-25T03:37:31,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/1ef5aea670304b458094730a7cbd0003 is 50, key is test_row_0/B:col10/1732505850723/Put/seqid=0 2024-11-25T03:37:31,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742024_1200 (size=12151) 2024-11-25T03:37:31,607 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/1ef5aea670304b458094730a7cbd0003 2024-11-25T03:37:31,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/711c92af6eee455d8c4fe512ce5b3cdb is 50, key is test_row_0/C:col10/1732505850723/Put/seqid=0 2024-11-25T03:37:31,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742025_1201 (size=12151) 2024-11-25T03:37:31,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-25T03:37:31,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:31,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. as already flushing 2024-11-25T03:37:31,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:31,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:31,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:31,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:31,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-25T03:37:31,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:31,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. as already flushing 2024-11-25T03:37:31,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:31,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:31,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:31,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:31,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-25T03:37:31,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:31,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505911918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:31,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:31,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505911918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:31,921 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:31,921 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:31,921 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:31,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:31,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:31,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:31,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:31,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505911927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:31,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:31,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505911927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:31,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:31,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:31,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:31,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:31,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:31,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:31,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-25T03:37:31,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:31,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. as already flushing 2024-11-25T03:37:31,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:31,951 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:31,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:31,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:32,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/711c92af6eee455d8c4fe512ce5b3cdb 2024-11-25T03:37:32,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/8431d0af668b4289927731f22e0ad62e as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/8431d0af668b4289927731f22e0ad62e 2024-11-25T03:37:32,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/8431d0af668b4289927731f22e0ad62e, entries=150, sequenceid=241, filesize=30.4 K 2024-11-25T03:37:32,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/1ef5aea670304b458094730a7cbd0003 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/1ef5aea670304b458094730a7cbd0003 2024-11-25T03:37:32,035 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/1ef5aea670304b458094730a7cbd0003, entries=150, sequenceid=241, filesize=11.9 K 2024-11-25T03:37:32,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/711c92af6eee455d8c4fe512ce5b3cdb as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/711c92af6eee455d8c4fe512ce5b3cdb 2024-11-25T03:37:32,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/711c92af6eee455d8c4fe512ce5b3cdb, entries=150, sequenceid=241, filesize=11.9 K 2024-11-25T03:37:32,041 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 9e284372599441cd748eba8961521ba1 in 1317ms, sequenceid=241, compaction requested=true 2024-11-25T03:37:32,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e284372599441cd748eba8961521ba1: 2024-11-25T03:37:32,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e284372599441cd748eba8961521ba1:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:37:32,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:32,041 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:37:32,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e284372599441cd748eba8961521ba1:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:37:32,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:32,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e284372599441cd748eba8961521ba1:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:37:32,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:37:32,041 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:37:32,042 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93759 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:37:32,042 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): 9e284372599441cd748eba8961521ba1/A is initiating minor compaction (all files) 2024-11-25T03:37:32,042 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e284372599441cd748eba8961521ba1/A in TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:32,042 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/72b08dfa6931432aa28e67ca5ff2927b, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/25185d89ded74a6abfe8d4af149cce41, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/8431d0af668b4289927731f22e0ad62e] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp, totalSize=91.6 K 2024-11-25T03:37:32,042 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:32,042 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. files: [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/72b08dfa6931432aa28e67ca5ff2927b, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/25185d89ded74a6abfe8d4af149cce41, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/8431d0af668b4289927731f22e0ad62e] 2024-11-25T03:37:32,043 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:37:32,043 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 9e284372599441cd748eba8961521ba1/B is initiating minor compaction (all files) 2024-11-25T03:37:32,043 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e284372599441cd748eba8961521ba1/B in TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:32,043 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72b08dfa6931432aa28e67ca5ff2927b, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732505847229 2024-11-25T03:37:32,043 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/5cbf16466bb84c0296892deebc5530e5, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/2c19961d7bd3438d96bd08f57b64f101, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/1ef5aea670304b458094730a7cbd0003] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp, totalSize=36.0 K 2024-11-25T03:37:32,043 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25185d89ded74a6abfe8d4af149cce41, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732505848382 2024-11-25T03:37:32,043 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8431d0af668b4289927731f22e0ad62e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732505850578 2024-11-25T03:37:32,044 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cbf16466bb84c0296892deebc5530e5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732505847229 2024-11-25T03:37:32,044 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c19961d7bd3438d96bd08f57b64f101, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732505848382 2024-11-25T03:37:32,044 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ef5aea670304b458094730a7cbd0003, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732505850578 2024-11-25T03:37:32,052 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9e284372599441cd748eba8961521ba1] 2024-11-25T03:37:32,054 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e284372599441cd748eba8961521ba1#B#compaction#171 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:37:32,055 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/73e15e802b214d4b98980af2a6e2a5cc is 50, key is test_row_0/B:col10/1732505850723/Put/seqid=0 2024-11-25T03:37:32,057 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112501344cd0fff9485882b956024749a2b4_9e284372599441cd748eba8961521ba1 store=[table=TestAcidGuarantees family=A region=9e284372599441cd748eba8961521ba1] 2024-11-25T03:37:32,058 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112501344cd0fff9485882b956024749a2b4_9e284372599441cd748eba8961521ba1, store=[table=TestAcidGuarantees family=A region=9e284372599441cd748eba8961521ba1] 2024-11-25T03:37:32,059 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112501344cd0fff9485882b956024749a2b4_9e284372599441cd748eba8961521ba1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9e284372599441cd748eba8961521ba1] 2024-11-25T03:37:32,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742026_1202 (size=12697) 2024-11-25T03:37:32,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742027_1203 (size=4469) 2024-11-25T03:37:32,074 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e284372599441cd748eba8961521ba1#A#compaction#170 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:37:32,075 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/2d6a767e83ad4ea59d7f8d8bee026b52 is 175, key is test_row_0/A:col10/1732505850723/Put/seqid=0 2024-11-25T03:37:32,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742028_1204 (size=31651) 2024-11-25T03:37:32,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-25T03:37:32,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:32,104 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 9e284372599441cd748eba8961521ba1 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-25T03:37:32,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125827d139107004cf39586cb5ed7284e94_9e284372599441cd748eba8961521ba1 is 50, key is test_row_0/A:col10/1732505850737/Put/seqid=0 2024-11-25T03:37:32,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742029_1205 (size=12304) 2024-11-25T03:37:32,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-25T03:37:32,472 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/73e15e802b214d4b98980af2a6e2a5cc as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/73e15e802b214d4b98980af2a6e2a5cc 2024-11-25T03:37:32,477 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e284372599441cd748eba8961521ba1/B of 9e284372599441cd748eba8961521ba1 into 73e15e802b214d4b98980af2a6e2a5cc(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:37:32,477 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e284372599441cd748eba8961521ba1: 2024-11-25T03:37:32,477 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., storeName=9e284372599441cd748eba8961521ba1/B, priority=13, startTime=1732505852041; duration=0sec 2024-11-25T03:37:32,477 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:37:32,477 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e284372599441cd748eba8961521ba1:B 2024-11-25T03:37:32,477 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:37:32,478 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:37:32,479 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 9e284372599441cd748eba8961521ba1/C is initiating minor compaction (all files) 2024-11-25T03:37:32,479 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e284372599441cd748eba8961521ba1/C in TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:32,479 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/e1327af6e0644ae09c114c28c5ecc1d2, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/c105947ef05a4ab3b1731c6d2e9faa7d, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/711c92af6eee455d8c4fe512ce5b3cdb] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp, totalSize=36.0 K 2024-11-25T03:37:32,479 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting e1327af6e0644ae09c114c28c5ecc1d2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732505847229 2024-11-25T03:37:32,479 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting c105947ef05a4ab3b1731c6d2e9faa7d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732505848382 2024-11-25T03:37:32,480 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 711c92af6eee455d8c4fe512ce5b3cdb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732505850578 2024-11-25T03:37:32,488 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/2d6a767e83ad4ea59d7f8d8bee026b52 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/2d6a767e83ad4ea59d7f8d8bee026b52 2024-11-25T03:37:32,490 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e284372599441cd748eba8961521ba1#C#compaction#173 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:37:32,491 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/83e7010886e1424cb8bff457b59a8117 is 50, key is test_row_0/C:col10/1732505850723/Put/seqid=0 2024-11-25T03:37:32,494 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e284372599441cd748eba8961521ba1/A of 9e284372599441cd748eba8961521ba1 into 2d6a767e83ad4ea59d7f8d8bee026b52(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:37:32,494 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e284372599441cd748eba8961521ba1: 2024-11-25T03:37:32,494 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., storeName=9e284372599441cd748eba8961521ba1/A, priority=13, startTime=1732505852041; duration=0sec 2024-11-25T03:37:32,494 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:32,494 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e284372599441cd748eba8961521ba1:A 2024-11-25T03:37:32,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742030_1206 (size=12697) 2024-11-25T03:37:32,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:37:32,520 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125827d139107004cf39586cb5ed7284e94_9e284372599441cd748eba8961521ba1 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125827d139107004cf39586cb5ed7284e94_9e284372599441cd748eba8961521ba1 2024-11-25T03:37:32,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/e7d70ce9033e4df6a0d165677873320b, store: [table=TestAcidGuarantees family=A region=9e284372599441cd748eba8961521ba1] 2024-11-25T03:37:32,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/e7d70ce9033e4df6a0d165677873320b is 175, key is test_row_0/A:col10/1732505850737/Put/seqid=0 2024-11-25T03:37:32,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742031_1207 (size=31105) 2024-11-25T03:37:32,530 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=251, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/e7d70ce9033e4df6a0d165677873320b 2024-11-25T03:37:32,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/4305116cd2e040b2ac475647a32db1dd is 50, key is test_row_0/B:col10/1732505850737/Put/seqid=0 2024-11-25T03:37:32,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742032_1208 (size=12151) 2024-11-25T03:37:32,902 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/83e7010886e1424cb8bff457b59a8117 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/83e7010886e1424cb8bff457b59a8117 2024-11-25T03:37:32,908 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e284372599441cd748eba8961521ba1/C of 9e284372599441cd748eba8961521ba1 into 83e7010886e1424cb8bff457b59a8117(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:37:32,908 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e284372599441cd748eba8961521ba1: 2024-11-25T03:37:32,908 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., storeName=9e284372599441cd748eba8961521ba1/C, priority=13, startTime=1732505852041; duration=0sec 2024-11-25T03:37:32,908 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:32,908 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e284372599441cd748eba8961521ba1:C 2024-11-25T03:37:32,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 9e284372599441cd748eba8961521ba1 2024-11-25T03:37:32,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. as already flushing 2024-11-25T03:37:32,945 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/4305116cd2e040b2ac475647a32db1dd 2024-11-25T03:37:32,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/1dd58621bd6049c8befd9fd249a2fa45 is 50, key is test_row_0/C:col10/1732505850737/Put/seqid=0 2024-11-25T03:37:32,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742033_1209 (size=12151) 2024-11-25T03:37:32,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:32,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505912962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:32,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:32,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:32,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:32,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:32,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505912963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:32,967 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:32,967 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:32,967 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:32,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:32,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505912965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:32,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:32,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:32,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:32,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:32,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505912966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:32,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:32,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:32,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:33,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:33,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505913077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:33,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:33,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:33,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:33,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:33,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505913077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:33,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:33,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:33,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:33,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:33,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505913077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:33,082 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:33,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:33,082 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:33,082 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:33,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505913077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:33,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:33,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:33,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:33,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:33,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505913287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:33,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:33,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505913287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:33,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:33,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:33,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:33,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505913287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:33,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:33,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:33,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:33,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:33,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:33,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505913287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:33,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:33,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:33,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:33,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:33,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:33,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:33,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-25T03:37:33,360 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/1dd58621bd6049c8befd9fd249a2fa45 2024-11-25T03:37:33,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/e7d70ce9033e4df6a0d165677873320b as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/e7d70ce9033e4df6a0d165677873320b 2024-11-25T03:37:33,370 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/e7d70ce9033e4df6a0d165677873320b, entries=150, sequenceid=251, filesize=30.4 K 2024-11-25T03:37:33,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/4305116cd2e040b2ac475647a32db1dd as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/4305116cd2e040b2ac475647a32db1dd 2024-11-25T03:37:33,376 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/4305116cd2e040b2ac475647a32db1dd, entries=150, sequenceid=251, filesize=11.9 K 2024-11-25T03:37:33,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/1dd58621bd6049c8befd9fd249a2fa45 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/1dd58621bd6049c8befd9fd249a2fa45 2024-11-25T03:37:33,384 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/1dd58621bd6049c8befd9fd249a2fa45, entries=150, sequenceid=251, filesize=11.9 K 2024-11-25T03:37:33,385 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 9e284372599441cd748eba8961521ba1 in 1281ms, sequenceid=251, compaction requested=false 2024-11-25T03:37:33,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 9e284372599441cd748eba8961521ba1: 2024-11-25T03:37:33,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:33,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-25T03:37:33,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-25T03:37:33,389 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-25T03:37:33,389 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2030 sec 2024-11-25T03:37:33,391 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 2.2110 sec 2024-11-25T03:37:33,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 9e284372599441cd748eba8961521ba1 2024-11-25T03:37:33,600 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:37:33,600 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e284372599441cd748eba8961521ba1 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-25T03:37:33,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:33,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505913603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:33,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:33,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:33,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:33,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505913605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:33,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:33,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:33,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:33,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:33,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:33,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505913606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:33,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:33,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:33,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:33,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:33,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505913607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:33,611 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:33,611 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:33,611 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:33,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125c511b59c956248dfa78378afbd3e1404_9e284372599441cd748eba8961521ba1 is 50, key is test_row_0/A:col10/1732505853600/Put/seqid=0 2024-11-25T03:37:33,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742034_1210 (size=12454) 2024-11-25T03:37:33,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:33,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505913717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:33,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:33,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:33,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:33,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505913717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:33,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:33,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:33,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:33,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:33,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505913717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:33,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:33,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:33,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:33,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:33,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:33,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505913717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:33,719 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:33,719 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:33,719 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:33,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:33,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505913927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:33,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:33,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505913927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:33,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:33,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505913927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:33,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:33,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505913927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:34,022 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:37:34,026 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125c511b59c956248dfa78378afbd3e1404_9e284372599441cd748eba8961521ba1 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125c511b59c956248dfa78378afbd3e1404_9e284372599441cd748eba8961521ba1 2024-11-25T03:37:34,027 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/8d75106762534279ba059fc686bfb893, store: [table=TestAcidGuarantees family=A region=9e284372599441cd748eba8961521ba1] 2024-11-25T03:37:34,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/8d75106762534279ba059fc686bfb893 is 175, key is test_row_0/A:col10/1732505853600/Put/seqid=0 2024-11-25T03:37:34,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742035_1211 (size=31255) 2024-11-25T03:37:34,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:34,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505914237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:34,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:34,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:34,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:34,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:34,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505914237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:34,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:34,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:34,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:34,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:34,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505914237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:34,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:34,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:34,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:34,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:34,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505914238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:34,242 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:34,242 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:34,242 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:34,432 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=282, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/8d75106762534279ba059fc686bfb893 2024-11-25T03:37:34,440 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/f0068b96da21451aa7bc75bee80a2530 is 50, key is test_row_0/B:col10/1732505853600/Put/seqid=0 2024-11-25T03:37:34,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742036_1212 (size=12301) 2024-11-25T03:37:34,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:34,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505914747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:34,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:34,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:34,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505914748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:34,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:34,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:34,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:34,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:34,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:34,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:34,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505914748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:34,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:34,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:34,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:34,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:34,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505914748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:34,751 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:34,751 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:34,751 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:34,845 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/f0068b96da21451aa7bc75bee80a2530 2024-11-25T03:37:34,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/44faa06b679743e49da24c15bce5c869 is 50, key is test_row_0/C:col10/1732505853600/Put/seqid=0 2024-11-25T03:37:34,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742037_1213 (size=12301) 2024-11-25T03:37:35,258 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/44faa06b679743e49da24c15bce5c869 2024-11-25T03:37:35,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/8d75106762534279ba059fc686bfb893 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/8d75106762534279ba059fc686bfb893 2024-11-25T03:37:35,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/8d75106762534279ba059fc686bfb893, entries=150, sequenceid=282, filesize=30.5 K 2024-11-25T03:37:35,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/f0068b96da21451aa7bc75bee80a2530 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/f0068b96da21451aa7bc75bee80a2530 2024-11-25T03:37:35,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/f0068b96da21451aa7bc75bee80a2530, entries=150, sequenceid=282, filesize=12.0 K 2024-11-25T03:37:35,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/44faa06b679743e49da24c15bce5c869 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/44faa06b679743e49da24c15bce5c869 2024-11-25T03:37:35,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/44faa06b679743e49da24c15bce5c869, entries=150, sequenceid=282, filesize=12.0 K 2024-11-25T03:37:35,278 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 9e284372599441cd748eba8961521ba1 in 1678ms, sequenceid=282, compaction requested=true 2024-11-25T03:37:35,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e284372599441cd748eba8961521ba1: 2024-11-25T03:37:35,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e284372599441cd748eba8961521ba1:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:37:35,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:35,279 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:37:35,279 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:37:35,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e284372599441cd748eba8961521ba1:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:37:35,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:35,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e284372599441cd748eba8961521ba1:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:37:35,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:37:35,280 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94011 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:37:35,280 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): 9e284372599441cd748eba8961521ba1/A is initiating minor compaction (all files) 2024-11-25T03:37:35,280 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e284372599441cd748eba8961521ba1/A in TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:35,280 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/2d6a767e83ad4ea59d7f8d8bee026b52, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/e7d70ce9033e4df6a0d165677873320b, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/8d75106762534279ba059fc686bfb893] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp, totalSize=91.8 K 2024-11-25T03:37:35,280 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:35,280 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. files: [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/2d6a767e83ad4ea59d7f8d8bee026b52, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/e7d70ce9033e4df6a0d165677873320b, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/8d75106762534279ba059fc686bfb893] 2024-11-25T03:37:35,281 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:37:35,281 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 9e284372599441cd748eba8961521ba1/B is initiating minor compaction (all files) 2024-11-25T03:37:35,281 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e284372599441cd748eba8961521ba1/B in TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:35,281 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/73e15e802b214d4b98980af2a6e2a5cc, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/4305116cd2e040b2ac475647a32db1dd, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/f0068b96da21451aa7bc75bee80a2530] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp, totalSize=36.3 K 2024-11-25T03:37:35,281 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d6a767e83ad4ea59d7f8d8bee026b52, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732505850578 2024-11-25T03:37:35,281 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 73e15e802b214d4b98980af2a6e2a5cc, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732505850578 2024-11-25T03:37:35,281 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7d70ce9033e4df6a0d165677873320b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732505850731 2024-11-25T03:37:35,281 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 4305116cd2e040b2ac475647a32db1dd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732505850731 2024-11-25T03:37:35,281 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d75106762534279ba059fc686bfb893, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732505852958 2024-11-25T03:37:35,282 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting f0068b96da21451aa7bc75bee80a2530, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732505852958 2024-11-25T03:37:35,288 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9e284372599441cd748eba8961521ba1] 2024-11-25T03:37:35,290 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125acae9a7988a948cf9dfc68b7e3fd8f19_9e284372599441cd748eba8961521ba1 store=[table=TestAcidGuarantees family=A region=9e284372599441cd748eba8961521ba1] 2024-11-25T03:37:35,291 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e284372599441cd748eba8961521ba1#B#compaction#180 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:37:35,292 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/79a155449d0a448fb814392c92f71af5 is 50, key is test_row_0/B:col10/1732505853600/Put/seqid=0 2024-11-25T03:37:35,292 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125acae9a7988a948cf9dfc68b7e3fd8f19_9e284372599441cd748eba8961521ba1, store=[table=TestAcidGuarantees family=A region=9e284372599441cd748eba8961521ba1] 2024-11-25T03:37:35,292 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125acae9a7988a948cf9dfc68b7e3fd8f19_9e284372599441cd748eba8961521ba1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9e284372599441cd748eba8961521ba1] 2024-11-25T03:37:35,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742038_1214 (size=12949) 2024-11-25T03:37:35,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742039_1215 (size=4469) 2024-11-25T03:37:35,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-25T03:37:35,348 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:37:35,348 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:37:35,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-25T03:37:35,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-25T03:37:35,350 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:37:35,350 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:37:35,351 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:37:35,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-25T03:37:35,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-25T03:37:35,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:35,503 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 9e284372599441cd748eba8961521ba1 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-25T03:37:35,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125d283d5fe32374436a85f2e46de100dba_9e284372599441cd748eba8961521ba1 is 50, key is test_row_0/A:col10/1732505853605/Put/seqid=0 2024-11-25T03:37:35,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742040_1216 (size=12454) 2024-11-25T03:37:35,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:37:35,520 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125d283d5fe32374436a85f2e46de100dba_9e284372599441cd748eba8961521ba1 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125d283d5fe32374436a85f2e46de100dba_9e284372599441cd748eba8961521ba1 2024-11-25T03:37:35,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/6e1809bb71364a13820e7027fb418a9c, store: [table=TestAcidGuarantees family=A region=9e284372599441cd748eba8961521ba1] 2024-11-25T03:37:35,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/6e1809bb71364a13820e7027fb418a9c is 175, key is test_row_0/A:col10/1732505853605/Put/seqid=0 2024-11-25T03:37:35,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742041_1217 (size=31255) 2024-11-25T03:37:35,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-25T03:37:35,703 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/79a155449d0a448fb814392c92f71af5 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/79a155449d0a448fb814392c92f71af5 2024-11-25T03:37:35,706 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e284372599441cd748eba8961521ba1#A#compaction#179 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:37:35,707 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/fbb7c6b6ab324512aa717638fb9152f8 is 175, key is test_row_0/A:col10/1732505853600/Put/seqid=0 2024-11-25T03:37:35,709 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e284372599441cd748eba8961521ba1/B of 9e284372599441cd748eba8961521ba1 into 79a155449d0a448fb814392c92f71af5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:37:35,709 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e284372599441cd748eba8961521ba1: 2024-11-25T03:37:35,709 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., storeName=9e284372599441cd748eba8961521ba1/B, priority=13, startTime=1732505855279; duration=0sec 2024-11-25T03:37:35,710 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:37:35,710 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e284372599441cd748eba8961521ba1:B 2024-11-25T03:37:35,710 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:37:35,711 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:37:35,711 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 9e284372599441cd748eba8961521ba1/C is initiating minor compaction (all files) 2024-11-25T03:37:35,711 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e284372599441cd748eba8961521ba1/C in TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:35,711 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/83e7010886e1424cb8bff457b59a8117, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/1dd58621bd6049c8befd9fd249a2fa45, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/44faa06b679743e49da24c15bce5c869] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp, totalSize=36.3 K 2024-11-25T03:37:35,712 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 83e7010886e1424cb8bff457b59a8117, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732505850578 2024-11-25T03:37:35,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742042_1218 (size=31903) 2024-11-25T03:37:35,712 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 1dd58621bd6049c8befd9fd249a2fa45, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732505850731 2024-11-25T03:37:35,713 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 44faa06b679743e49da24c15bce5c869, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732505852958 2024-11-25T03:37:35,720 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e284372599441cd748eba8961521ba1#C#compaction#182 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:37:35,721 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/4121132e26634f91b1f42364eae4723a is 50, key is test_row_0/C:col10/1732505853600/Put/seqid=0 2024-11-25T03:37:35,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742043_1219 (size=12949) 2024-11-25T03:37:35,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 9e284372599441cd748eba8961521ba1 2024-11-25T03:37:35,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. as already flushing 2024-11-25T03:37:35,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:35,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505915792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:35,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:35,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:35,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:35,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:35,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505915793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:35,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:35,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:35,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:35,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:35,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505915794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:35,796 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:35,796 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:35,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:35,796 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:35,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505915794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:35,797 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:35,797 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:35,797 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:35,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:35,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505915897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:35,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:35,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:35,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:35,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:35,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505915907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:35,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:35,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:35,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:35,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505915907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:35,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:35,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:35,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:35,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505915907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:35,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:35,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:35,911 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:35,911 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:35,911 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:35,927 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=290, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/6e1809bb71364a13820e7027fb418a9c 2024-11-25T03:37:35,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/d5a7586ee71c435b9755e2b67cf58f58 is 50, key is test_row_0/B:col10/1732505853605/Put/seqid=0 2024-11-25T03:37:35,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742044_1220 (size=12301) 2024-11-25T03:37:35,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-25T03:37:36,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:36,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505916107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:36,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:36,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:36,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:36,118 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/fbb7c6b6ab324512aa717638fb9152f8 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/fbb7c6b6ab324512aa717638fb9152f8 2024-11-25T03:37:36,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:36,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505916118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:36,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:36,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:36,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:36,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:36,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505916118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:36,120 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:36,120 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:36,120 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:36,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:36,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505916118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:36,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:36,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:36,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:36,124 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e284372599441cd748eba8961521ba1/A of 9e284372599441cd748eba8961521ba1 into fbb7c6b6ab324512aa717638fb9152f8(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:37:36,124 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e284372599441cd748eba8961521ba1: 2024-11-25T03:37:36,124 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., storeName=9e284372599441cd748eba8961521ba1/A, priority=13, startTime=1732505855278; duration=0sec 2024-11-25T03:37:36,124 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:36,124 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e284372599441cd748eba8961521ba1:A 2024-11-25T03:37:36,131 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/4121132e26634f91b1f42364eae4723a as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/4121132e26634f91b1f42364eae4723a 2024-11-25T03:37:36,136 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e284372599441cd748eba8961521ba1/C of 9e284372599441cd748eba8961521ba1 into 4121132e26634f91b1f42364eae4723a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:37:36,136 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e284372599441cd748eba8961521ba1: 2024-11-25T03:37:36,136 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., storeName=9e284372599441cd748eba8961521ba1/C, priority=13, startTime=1732505855279; duration=0sec 2024-11-25T03:37:36,137 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:36,137 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e284372599441cd748eba8961521ba1:C 2024-11-25T03:37:36,342 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/d5a7586ee71c435b9755e2b67cf58f58 2024-11-25T03:37:36,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/ccf7532908b648aa8220cbc254e33e65 is 50, key is test_row_0/C:col10/1732505853605/Put/seqid=0 2024-11-25T03:37:36,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742045_1221 (size=12301) 2024-11-25T03:37:36,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:36,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505916417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:36,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:36,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:36,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:36,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:36,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505916427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:36,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:36,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:36,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:36,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:36,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505916427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:36,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:36,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:36,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:36,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:36,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505916427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:36,429 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:36,429 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:36,429 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:36,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-25T03:37:36,754 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/ccf7532908b648aa8220cbc254e33e65 2024-11-25T03:37:36,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/6e1809bb71364a13820e7027fb418a9c as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/6e1809bb71364a13820e7027fb418a9c 2024-11-25T03:37:36,765 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/6e1809bb71364a13820e7027fb418a9c, entries=150, sequenceid=290, filesize=30.5 K 2024-11-25T03:37:36,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/d5a7586ee71c435b9755e2b67cf58f58 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/d5a7586ee71c435b9755e2b67cf58f58 2024-11-25T03:37:36,771 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/d5a7586ee71c435b9755e2b67cf58f58, entries=150, sequenceid=290, filesize=12.0 K 2024-11-25T03:37:36,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/ccf7532908b648aa8220cbc254e33e65 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/ccf7532908b648aa8220cbc254e33e65 2024-11-25T03:37:36,777 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/ccf7532908b648aa8220cbc254e33e65, entries=150, sequenceid=290, filesize=12.0 K 2024-11-25T03:37:36,778 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=174.43 KB/178620 for 9e284372599441cd748eba8961521ba1 in 1275ms, sequenceid=290, compaction requested=false 2024-11-25T03:37:36,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 9e284372599441cd748eba8961521ba1: 2024-11-25T03:37:36,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:36,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-25T03:37:36,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-25T03:37:36,781 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-25T03:37:36,781 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4280 sec 2024-11-25T03:37:36,782 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 1.4330 sec 2024-11-25T03:37:36,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 9e284372599441cd748eba8961521ba1 2024-11-25T03:37:36,932 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:37:36,932 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e284372599441cd748eba8961521ba1 3/3 column families, dataSize=181.14 KB heapSize=475.36 KB 2024-11-25T03:37:36,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125801d7b90fd0241eab153492c288934f7_9e284372599441cd748eba8961521ba1 is 50, key is test_row_0/A:col10/1732505855791/Put/seqid=0 2024-11-25T03:37:36,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:36,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505916938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:36,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:36,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505916940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:36,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:36,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505916943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:36,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:36,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:36,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:36,944 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:36,944 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:36,944 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:36,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:36,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505916944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:36,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:36,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:36,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:36,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:36,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:36,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:36,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742046_1222 (size=14994) 2024-11-25T03:37:36,946 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:37:36,951 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125801d7b90fd0241eab153492c288934f7_9e284372599441cd748eba8961521ba1 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125801d7b90fd0241eab153492c288934f7_9e284372599441cd748eba8961521ba1 2024-11-25T03:37:36,952 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/e06f57f3b0a34de49f4c6c0879cf7265, store: [table=TestAcidGuarantees family=A region=9e284372599441cd748eba8961521ba1] 2024-11-25T03:37:36,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/e06f57f3b0a34de49f4c6c0879cf7265 is 175, key is test_row_0/A:col10/1732505855791/Put/seqid=0 2024-11-25T03:37:36,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742047_1223 (size=39949) 2024-11-25T03:37:36,959 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=323, memsize=60.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/e06f57f3b0a34de49f4c6c0879cf7265 2024-11-25T03:37:36,968 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/aad98ab33d754328ae4f693263a5d6a8 is 50, key is test_row_0/B:col10/1732505855791/Put/seqid=0 2024-11-25T03:37:36,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742048_1224 (size=12301) 2024-11-25T03:37:37,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:37,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505917047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:37,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:37,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:37,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:37,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505917048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:37,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:37,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:37,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505917048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:37,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:37,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:37,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:37,050 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:37,051 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:37,051 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:37,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:37,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505917257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:37,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:37,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:37,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:37,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:37,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505917257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:37,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:37,262 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:37,262 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:37,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505917257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:37,262 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:37,262 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:37,262 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:37,262 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:37,373 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/aad98ab33d754328ae4f693263a5d6a8 2024-11-25T03:37:37,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/e078e2129a0d4b9bbc8e85629cb42d10 is 50, key is test_row_0/C:col10/1732505855791/Put/seqid=0 2024-11-25T03:37:37,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742049_1225 (size=12301) 2024-11-25T03:37:37,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-25T03:37:37,498 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:37:37,498 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:37:37,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-25T03:37:37,500 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:37:37,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-25T03:37:37,501 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:37:37,501 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:37:37,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:37,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:37,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505917568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:37,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505917568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:37,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:37,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505917568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:37,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:37,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:37,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:37,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:37,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:37,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:37,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:37,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:37,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:37,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-25T03:37:37,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-25T03:37:37,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:37,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. as already flushing 2024-11-25T03:37:37,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:37,655 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:37,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:37,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:37,792 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/e078e2129a0d4b9bbc8e85629cb42d10 2024-11-25T03:37:37,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/e06f57f3b0a34de49f4c6c0879cf7265 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/e06f57f3b0a34de49f4c6c0879cf7265 2024-11-25T03:37:37,804 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/e06f57f3b0a34de49f4c6c0879cf7265, entries=200, sequenceid=323, filesize=39.0 K 2024-11-25T03:37:37,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/aad98ab33d754328ae4f693263a5d6a8 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/aad98ab33d754328ae4f693263a5d6a8 2024-11-25T03:37:37,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-25T03:37:37,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:37,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. as already flushing 2024-11-25T03:37:37,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:37,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:37,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:37,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:37,815 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/aad98ab33d754328ae4f693263a5d6a8, entries=150, sequenceid=323, filesize=12.0 K 2024-11-25T03:37:37,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/e078e2129a0d4b9bbc8e85629cb42d10 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/e078e2129a0d4b9bbc8e85629cb42d10 2024-11-25T03:37:37,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-25T03:37:37,823 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/e078e2129a0d4b9bbc8e85629cb42d10, entries=150, sequenceid=323, filesize=12.0 K 2024-11-25T03:37:37,824 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=26.84 KB/27480 for 9e284372599441cd748eba8961521ba1 in 892ms, sequenceid=323, compaction requested=true 2024-11-25T03:37:37,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e284372599441cd748eba8961521ba1: 2024-11-25T03:37:37,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e284372599441cd748eba8961521ba1:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:37:37,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:37,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e284372599441cd748eba8961521ba1:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:37:37,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:37,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e284372599441cd748eba8961521ba1:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:37:37,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:37:37,824 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:37:37,824 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:37:37,825 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:37:37,825 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): 9e284372599441cd748eba8961521ba1/A is initiating minor compaction (all files) 2024-11-25T03:37:37,825 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e284372599441cd748eba8961521ba1/A in TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:37,826 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/fbb7c6b6ab324512aa717638fb9152f8, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/6e1809bb71364a13820e7027fb418a9c, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/e06f57f3b0a34de49f4c6c0879cf7265] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp, totalSize=100.7 K 2024-11-25T03:37:37,826 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:37,826 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. files: [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/fbb7c6b6ab324512aa717638fb9152f8, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/6e1809bb71364a13820e7027fb418a9c, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/e06f57f3b0a34de49f4c6c0879cf7265] 2024-11-25T03:37:37,826 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:37:37,826 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 9e284372599441cd748eba8961521ba1/B is initiating minor compaction (all files) 2024-11-25T03:37:37,826 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbb7c6b6ab324512aa717638fb9152f8, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732505852958 2024-11-25T03:37:37,826 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e284372599441cd748eba8961521ba1/B in TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:37,826 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/79a155449d0a448fb814392c92f71af5, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/d5a7586ee71c435b9755e2b67cf58f58, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/aad98ab33d754328ae4f693263a5d6a8] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp, totalSize=36.7 K 2024-11-25T03:37:37,826 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e1809bb71364a13820e7027fb418a9c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732505853602 2024-11-25T03:37:37,827 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 79a155449d0a448fb814392c92f71af5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732505852958 2024-11-25T03:37:37,827 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting e06f57f3b0a34de49f4c6c0879cf7265, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1732505855791 2024-11-25T03:37:37,827 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting d5a7586ee71c435b9755e2b67cf58f58, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732505853602 2024-11-25T03:37:37,827 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting aad98ab33d754328ae4f693263a5d6a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1732505855791 2024-11-25T03:37:37,838 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e284372599441cd748eba8961521ba1#B#compaction#188 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:37:37,839 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/ac9fc295181c4badaf59a7313283fbed is 50, key is test_row_0/B:col10/1732505855791/Put/seqid=0 2024-11-25T03:37:37,839 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9e284372599441cd748eba8961521ba1] 2024-11-25T03:37:37,843 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411251df8bf40c5064de396d48845f0d90932_9e284372599441cd748eba8961521ba1 store=[table=TestAcidGuarantees family=A region=9e284372599441cd748eba8961521ba1] 2024-11-25T03:37:37,845 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411251df8bf40c5064de396d48845f0d90932_9e284372599441cd748eba8961521ba1, store=[table=TestAcidGuarantees family=A region=9e284372599441cd748eba8961521ba1] 2024-11-25T03:37:37,845 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411251df8bf40c5064de396d48845f0d90932_9e284372599441cd748eba8961521ba1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9e284372599441cd748eba8961521ba1] 2024-11-25T03:37:37,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742050_1226 (size=13051) 2024-11-25T03:37:37,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742051_1227 (size=4469) 2024-11-25T03:37:37,871 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/ac9fc295181c4badaf59a7313283fbed as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/ac9fc295181c4badaf59a7313283fbed 2024-11-25T03:37:37,873 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e284372599441cd748eba8961521ba1#A#compaction#189 average throughput is 0.74 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:37:37,873 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/cebcb790eaf8497fa86cf2798a5292ee is 175, key is test_row_0/A:col10/1732505855791/Put/seqid=0 2024-11-25T03:37:37,878 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e284372599441cd748eba8961521ba1/B of 9e284372599441cd748eba8961521ba1 into ac9fc295181c4badaf59a7313283fbed(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:37:37,878 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e284372599441cd748eba8961521ba1: 2024-11-25T03:37:37,878 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., storeName=9e284372599441cd748eba8961521ba1/B, priority=13, startTime=1732505857824; duration=0sec 2024-11-25T03:37:37,878 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:37:37,878 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e284372599441cd748eba8961521ba1:B 2024-11-25T03:37:37,878 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:37:37,880 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:37:37,880 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 9e284372599441cd748eba8961521ba1/C is initiating minor compaction (all files) 2024-11-25T03:37:37,880 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e284372599441cd748eba8961521ba1/C in TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:37,880 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/4121132e26634f91b1f42364eae4723a, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/ccf7532908b648aa8220cbc254e33e65, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/e078e2129a0d4b9bbc8e85629cb42d10] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp, totalSize=36.7 K 2024-11-25T03:37:37,881 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 4121132e26634f91b1f42364eae4723a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732505852958 2024-11-25T03:37:37,881 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting ccf7532908b648aa8220cbc254e33e65, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732505853602 2024-11-25T03:37:37,882 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting e078e2129a0d4b9bbc8e85629cb42d10, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1732505855791 2024-11-25T03:37:37,894 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e284372599441cd748eba8961521ba1#C#compaction#190 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:37:37,895 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/6c043195829c490da48352c136d350fb is 50, key is test_row_0/C:col10/1732505855791/Put/seqid=0 2024-11-25T03:37:37,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742052_1228 (size=32005) 2024-11-25T03:37:37,902 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/cebcb790eaf8497fa86cf2798a5292ee as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/cebcb790eaf8497fa86cf2798a5292ee 2024-11-25T03:37:37,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742053_1229 (size=13051) 2024-11-25T03:37:37,910 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/6c043195829c490da48352c136d350fb as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/6c043195829c490da48352c136d350fb 2024-11-25T03:37:37,910 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e284372599441cd748eba8961521ba1/A of 9e284372599441cd748eba8961521ba1 into cebcb790eaf8497fa86cf2798a5292ee(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:37:37,910 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e284372599441cd748eba8961521ba1: 2024-11-25T03:37:37,910 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., storeName=9e284372599441cd748eba8961521ba1/A, priority=13, startTime=1732505857824; duration=0sec 2024-11-25T03:37:37,910 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:37,910 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e284372599441cd748eba8961521ba1:A 2024-11-25T03:37:37,915 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e284372599441cd748eba8961521ba1/C of 9e284372599441cd748eba8961521ba1 into 6c043195829c490da48352c136d350fb(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:37:37,915 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e284372599441cd748eba8961521ba1: 2024-11-25T03:37:37,915 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., storeName=9e284372599441cd748eba8961521ba1/C, priority=13, startTime=1732505857824; duration=0sec 2024-11-25T03:37:37,916 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:37,916 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e284372599441cd748eba8961521ba1:C 2024-11-25T03:37:37,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-25T03:37:37,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:37,964 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 9e284372599441cd748eba8961521ba1 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-25T03:37:37,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. as already flushing 2024-11-25T03:37:37,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 9e284372599441cd748eba8961521ba1 2024-11-25T03:37:37,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112573d84be70abd49f58705bcc55592bebe_9e284372599441cd748eba8961521ba1 is 50, key is test_row_0/A:col10/1732505857960/Put/seqid=0 2024-11-25T03:37:38,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742054_1230 (size=14994) 2024-11-25T03:37:38,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:38,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505918081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:38,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:38,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:38,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:38,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:38,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505918083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:38,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:38,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505918083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:38,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:38,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:38,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:38,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:38,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505918083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:38,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:38,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:38,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:38,091 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:38,091 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:38,091 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:38,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-25T03:37:38,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:38,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505918187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:38,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:38,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:38,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:38,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:38,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:38,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505918197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:38,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505918197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:38,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:38,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:38,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:38,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:38,198 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:38,198 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:38,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505918198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:38,198 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:38,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:38,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:38,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:38,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:37:38,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:38,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505918397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:38,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:38,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:38,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:38,405 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112573d84be70abd49f58705bcc55592bebe_9e284372599441cd748eba8961521ba1 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112573d84be70abd49f58705bcc55592bebe_9e284372599441cd748eba8961521ba1 2024-11-25T03:37:38,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/d74739bf27334d3894e0cc514dddecb9, store: [table=TestAcidGuarantees family=A region=9e284372599441cd748eba8961521ba1] 2024-11-25T03:37:38,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/d74739bf27334d3894e0cc514dddecb9 is 175, key is test_row_0/A:col10/1732505857960/Put/seqid=0 2024-11-25T03:37:38,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:38,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505918408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:38,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:38,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505918408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:38,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:38,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505918408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:38,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:38,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:38,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:38,410 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:38,410 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:38,410 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:38,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:38,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:38,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:38,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742055_1231 (size=39949) 2024-11-25T03:37:38,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-25T03:37:38,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:38,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505918707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:38,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:38,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:38,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:38,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:38,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505918717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:38,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:38,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505918717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:38,720 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:38,720 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:38,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:38,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:38,720 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:38,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:38,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505918717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:38,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:38,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:38,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:38,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:38,814 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=336, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/d74739bf27334d3894e0cc514dddecb9 2024-11-25T03:37:38,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/9985f7442e7b402db900fa2fbc8b7edb is 50, key is test_row_0/B:col10/1732505857960/Put/seqid=0 2024-11-25T03:37:38,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742056_1232 (size=12301) 2024-11-25T03:37:38,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:38,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53018 deadline: 1732505918848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:38,849 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,34881,1732505806588 for 'test_row_2' in 9e284372599441cd748eba8961521ba1 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18274 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:37:38,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:38,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:38,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:39,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:39,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505919217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:39,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:39,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:39,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:39,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:39,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:39,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505919228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:39,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505919228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:39,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:39,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:39,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505919228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:39,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:39,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:39,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:39,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:39,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:39,232 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:39,232 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:39,232 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:39,237 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/9985f7442e7b402db900fa2fbc8b7edb 2024-11-25T03:37:39,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/42a915331a134593953a4df1473e60dd is 50, key is test_row_0/C:col10/1732505857960/Put/seqid=0 2024-11-25T03:37:39,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742057_1233 (size=12301) 2024-11-25T03:37:39,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-25T03:37:39,654 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/42a915331a134593953a4df1473e60dd 2024-11-25T03:37:39,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/d74739bf27334d3894e0cc514dddecb9 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/d74739bf27334d3894e0cc514dddecb9 2024-11-25T03:37:39,664 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/d74739bf27334d3894e0cc514dddecb9, entries=200, sequenceid=336, filesize=39.0 K 2024-11-25T03:37:39,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/9985f7442e7b402db900fa2fbc8b7edb as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/9985f7442e7b402db900fa2fbc8b7edb 2024-11-25T03:37:39,670 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/9985f7442e7b402db900fa2fbc8b7edb, entries=150, sequenceid=336, filesize=12.0 K 2024-11-25T03:37:39,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/42a915331a134593953a4df1473e60dd as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/42a915331a134593953a4df1473e60dd 2024-11-25T03:37:39,675 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/42a915331a134593953a4df1473e60dd, entries=150, sequenceid=336, filesize=12.0 K 2024-11-25T03:37:39,676 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=167.72 KB/171750 for 9e284372599441cd748eba8961521ba1 in 1712ms, sequenceid=336, compaction requested=false 2024-11-25T03:37:39,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 9e284372599441cd748eba8961521ba1: 2024-11-25T03:37:39,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:39,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-25T03:37:39,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-25T03:37:39,679 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-25T03:37:39,679 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1760 sec 2024-11-25T03:37:39,681 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 2.1810 sec 2024-11-25T03:37:40,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 9e284372599441cd748eba8961521ba1 2024-11-25T03:37:40,240 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:37:40,240 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e284372599441cd748eba8961521ba1 3/3 column families, dataSize=181.14 KB heapSize=475.36 KB 2024-11-25T03:37:40,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:40,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53032 deadline: 1732505920248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:40,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:40,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505920250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:40,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:40,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505920252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:40,255 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:40,255 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:40,255 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:40,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:40,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:40,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:40,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505920252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:40,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:40,255 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:40,255 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:40,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:40,256 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,34881,1732505806588 for 'test_row_1' in 9e284372599441cd748eba8961521ba1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:37:40,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:40,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:40,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:40,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112559235a56703f490f9053a3879a2b7598_9e284372599441cd748eba8961521ba1 is 50, key is test_row_0/A:col10/1732505858081/Put/seqid=0 2024-11-25T03:37:40,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742058_1234 (size=12454) 2024-11-25T03:37:40,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:40,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505920358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:40,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:40,359 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:40,359 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:40,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505920358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:40,359 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:40,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:40,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505920358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:40,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:40,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:40,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:40,360 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:40,360 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:40,360 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:40,454 INFO [Thread-689 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-689. 2024-11-25T03:37:40,454 DEBUG [Thread-689 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:37:40,454 DEBUG [Thread-689 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:40,454 DEBUG [Thread-689 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:40,454 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:37:40,455 INFO [Thread-695 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-695. 2024-11-25T03:37:40,455 DEBUG [Thread-695 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:37:40,455 DEBUG [Thread-695 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:40,455 DEBUG [Thread-695 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:40,455 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:37:40,456 INFO [Thread-691 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-691. 2024-11-25T03:37:40,456 DEBUG [Thread-691 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:37:40,456 DEBUG [Thread-691 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:40,456 DEBUG [Thread-691 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:40,456 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:37:40,458 INFO [Thread-693 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-693. 2024-11-25T03:37:40,458 DEBUG [Thread-693 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:37:40,458 DEBUG [Thread-693 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:40,458 DEBUG [Thread-693 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:40,458 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:37:40,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:40,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:40,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:40,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505920567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:40,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505920567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:40,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505920568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:40,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:40,568 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:40,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:40,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:40,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:40,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:40,568 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:40,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:40,568 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:40,665 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:37:40,669 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112559235a56703f490f9053a3879a2b7598_9e284372599441cd748eba8961521ba1 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112559235a56703f490f9053a3879a2b7598_9e284372599441cd748eba8961521ba1 2024-11-25T03:37:40,669 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/4a98f6fb93c04a80aa8cd1f6d7873f37, store: [table=TestAcidGuarantees family=A region=9e284372599441cd748eba8961521ba1] 2024-11-25T03:37:40,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/4a98f6fb93c04a80aa8cd1f6d7873f37 is 175, key is test_row_0/A:col10/1732505858081/Put/seqid=0 2024-11-25T03:37:40,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742059_1235 (size=31255) 2024-11-25T03:37:40,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:40,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:40,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:40,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505920878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:40,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505920878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:40,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505920878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:40,879 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:40,879 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:40,879 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:40,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:40,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:40,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:40,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:40,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:40,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:41,074 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=367, memsize=62.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/4a98f6fb93c04a80aa8cd1f6d7873f37 2024-11-25T03:37:41,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/366af6ef0b22402bbd0940bb3d684b68 is 50, key is test_row_0/B:col10/1732505858081/Put/seqid=0 2024-11-25T03:37:41,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742060_1236 (size=12301) 2024-11-25T03:37:41,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:41,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:41,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:41,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732505921387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:41,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52976 deadline: 1732505921387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:41,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732505921387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:41,388 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:41,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:41,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:41,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:41,388 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:41,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:41,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:41,388 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:41,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:37:41,485 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/366af6ef0b22402bbd0940bb3d684b68 2024-11-25T03:37:41,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/473a1f658ca8429d99ac4fa21457f84d is 50, key is test_row_0/C:col10/1732505858081/Put/seqid=0 2024-11-25T03:37:41,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742061_1237 (size=12301) 2024-11-25T03:37:41,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-25T03:37:41,668 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:37:41,897 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/473a1f658ca8429d99ac4fa21457f84d 2024-11-25T03:37:41,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/4a98f6fb93c04a80aa8cd1f6d7873f37 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/4a98f6fb93c04a80aa8cd1f6d7873f37 2024-11-25T03:37:41,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/4a98f6fb93c04a80aa8cd1f6d7873f37, entries=150, sequenceid=367, filesize=30.5 K 2024-11-25T03:37:41,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/366af6ef0b22402bbd0940bb3d684b68 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/366af6ef0b22402bbd0940bb3d684b68 2024-11-25T03:37:41,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/366af6ef0b22402bbd0940bb3d684b68, entries=150, sequenceid=367, filesize=12.0 K 2024-11-25T03:37:41,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/473a1f658ca8429d99ac4fa21457f84d as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/473a1f658ca8429d99ac4fa21457f84d 2024-11-25T03:37:41,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/473a1f658ca8429d99ac4fa21457f84d, entries=150, sequenceid=367, filesize=12.0 K 2024-11-25T03:37:41,912 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~187.85 KB/192360, heapSize ~492.89 KB/504720, currentSize=13.42 KB/13740 for 9e284372599441cd748eba8961521ba1 in 1672ms, sequenceid=367, compaction requested=true 2024-11-25T03:37:41,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e284372599441cd748eba8961521ba1: 2024-11-25T03:37:41,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e284372599441cd748eba8961521ba1:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:37:41,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:41,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e284372599441cd748eba8961521ba1:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:37:41,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:41,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e284372599441cd748eba8961521ba1:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:37:41,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:37:41,912 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:37:41,912 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:37:41,913 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:37:41,913 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 9e284372599441cd748eba8961521ba1/B is initiating minor compaction (all files) 2024-11-25T03:37:41,913 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:37:41,913 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e284372599441cd748eba8961521ba1/B in TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:41,913 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): 9e284372599441cd748eba8961521ba1/A is initiating minor compaction (all files) 2024-11-25T03:37:41,913 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/ac9fc295181c4badaf59a7313283fbed, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/9985f7442e7b402db900fa2fbc8b7edb, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/366af6ef0b22402bbd0940bb3d684b68] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp, totalSize=36.8 K 2024-11-25T03:37:41,913 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e284372599441cd748eba8961521ba1/A in TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:41,913 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/cebcb790eaf8497fa86cf2798a5292ee, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/d74739bf27334d3894e0cc514dddecb9, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/4a98f6fb93c04a80aa8cd1f6d7873f37] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp, totalSize=100.8 K 2024-11-25T03:37:41,913 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:41,913 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. files: [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/cebcb790eaf8497fa86cf2798a5292ee, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/d74739bf27334d3894e0cc514dddecb9, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/4a98f6fb93c04a80aa8cd1f6d7873f37] 2024-11-25T03:37:41,913 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting ac9fc295181c4badaf59a7313283fbed, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1732505855791 2024-11-25T03:37:41,914 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting cebcb790eaf8497fa86cf2798a5292ee, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1732505855791 2024-11-25T03:37:41,914 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 9985f7442e7b402db900fa2fbc8b7edb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732505856937 2024-11-25T03:37:41,914 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting d74739bf27334d3894e0cc514dddecb9, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732505856932 2024-11-25T03:37:41,914 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 366af6ef0b22402bbd0940bb3d684b68, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732505858081 2024-11-25T03:37:41,914 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a98f6fb93c04a80aa8cd1f6d7873f37, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732505858081 2024-11-25T03:37:41,921 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e284372599441cd748eba8961521ba1#B#compaction#197 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:37:41,922 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/0dd0135b9c3744ddafefaff6e0ab2efa is 50, key is test_row_0/B:col10/1732505858081/Put/seqid=0 2024-11-25T03:37:41,924 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9e284372599441cd748eba8961521ba1] 2024-11-25T03:37:41,929 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112580bcc10ea5684a58aec39b4a579e9ad1_9e284372599441cd748eba8961521ba1 store=[table=TestAcidGuarantees family=A region=9e284372599441cd748eba8961521ba1] 2024-11-25T03:37:41,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742062_1238 (size=13153) 2024-11-25T03:37:41,937 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/0dd0135b9c3744ddafefaff6e0ab2efa as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/0dd0135b9c3744ddafefaff6e0ab2efa 2024-11-25T03:37:41,944 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e284372599441cd748eba8961521ba1/B of 9e284372599441cd748eba8961521ba1 into 0dd0135b9c3744ddafefaff6e0ab2efa(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:37:41,944 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e284372599441cd748eba8961521ba1: 2024-11-25T03:37:41,944 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., storeName=9e284372599441cd748eba8961521ba1/B, priority=13, startTime=1732505861912; duration=0sec 2024-11-25T03:37:41,944 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:37:41,944 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e284372599441cd748eba8961521ba1:B 2024-11-25T03:37:41,944 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:37:41,945 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:37:41,945 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 9e284372599441cd748eba8961521ba1/C is initiating minor compaction (all files) 2024-11-25T03:37:41,945 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e284372599441cd748eba8961521ba1/C in TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:41,945 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/6c043195829c490da48352c136d350fb, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/42a915331a134593953a4df1473e60dd, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/473a1f658ca8429d99ac4fa21457f84d] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp, totalSize=36.8 K 2024-11-25T03:37:41,946 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c043195829c490da48352c136d350fb, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1732505855791 2024-11-25T03:37:41,946 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 42a915331a134593953a4df1473e60dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732505856937 2024-11-25T03:37:41,947 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 473a1f658ca8429d99ac4fa21457f84d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732505858081 2024-11-25T03:37:41,950 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112580bcc10ea5684a58aec39b4a579e9ad1_9e284372599441cd748eba8961521ba1, store=[table=TestAcidGuarantees family=A region=9e284372599441cd748eba8961521ba1] 2024-11-25T03:37:41,950 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112580bcc10ea5684a58aec39b4a579e9ad1_9e284372599441cd748eba8961521ba1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9e284372599441cd748eba8961521ba1] 2024-11-25T03:37:41,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742063_1239 (size=4469) 2024-11-25T03:37:41,959 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e284372599441cd748eba8961521ba1#A#compaction#198 average throughput is 0.70 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:37:41,960 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/6f889c4fb9d146e9b551e01c3e15c4dc is 175, key is test_row_0/A:col10/1732505858081/Put/seqid=0 2024-11-25T03:37:41,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742064_1240 (size=32107) 2024-11-25T03:37:41,964 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e284372599441cd748eba8961521ba1#C#compaction#199 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:37:41,965 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/df55f2b7d2fa40afb33206a841c44988 is 50, key is test_row_0/C:col10/1732505858081/Put/seqid=0 2024-11-25T03:37:41,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742065_1241 (size=13153) 2024-11-25T03:37:41,973 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/df55f2b7d2fa40afb33206a841c44988 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/df55f2b7d2fa40afb33206a841c44988 2024-11-25T03:37:41,978 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e284372599441cd748eba8961521ba1/C of 9e284372599441cd748eba8961521ba1 into df55f2b7d2fa40afb33206a841c44988(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:37:41,978 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e284372599441cd748eba8961521ba1: 2024-11-25T03:37:41,978 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., storeName=9e284372599441cd748eba8961521ba1/C, priority=13, startTime=1732505861912; duration=0sec 2024-11-25T03:37:41,978 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:41,978 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e284372599441cd748eba8961521ba1:C 2024-11-25T03:37:42,269 INFO [Thread-682 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-682. 2024-11-25T03:37:42,269 DEBUG [Thread-682 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:37:42,269 DEBUG [Thread-682 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:42,269 DEBUG [Thread-682 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:42,269 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:37:42,369 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/6f889c4fb9d146e9b551e01c3e15c4dc as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/6f889c4fb9d146e9b551e01c3e15c4dc 2024-11-25T03:37:42,374 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e284372599441cd748eba8961521ba1/A of 9e284372599441cd748eba8961521ba1 into 6f889c4fb9d146e9b551e01c3e15c4dc(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:37:42,374 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e284372599441cd748eba8961521ba1: 2024-11-25T03:37:42,374 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., storeName=9e284372599441cd748eba8961521ba1/A, priority=13, startTime=1732505861912; duration=0sec 2024-11-25T03:37:42,374 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:42,374 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e284372599441cd748eba8961521ba1:A 2024-11-25T03:37:42,399 INFO [Thread-678 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-678. 2024-11-25T03:37:42,399 INFO [Thread-684 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-684. 2024-11-25T03:37:42,399 DEBUG [Thread-678 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:37:42,399 INFO [Thread-686 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-686. 2024-11-25T03:37:42,399 DEBUG [Thread-684 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:37:42,399 DEBUG [Thread-678 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:42,399 DEBUG [Thread-684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:42,399 DEBUG [Thread-686 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:37:42,399 DEBUG [Thread-686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:42,399 DEBUG [Thread-678 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:42,399 DEBUG [Thread-686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:42,399 DEBUG [Thread-684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:42,399 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:37:42,399 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:37:42,399 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:37:44,798 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T03:37:48,870 INFO [Thread-680 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-680. 2024-11-25T03:37:48,870 DEBUG [Thread-680 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:37:48,870 DEBUG [Thread-680 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:48,870 DEBUG [Thread-680 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:48,870 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:37:48,871 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-25T03:37:48,871 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-25T03:37:48,871 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 5 2024-11-25T03:37:48,871 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-25T03:37:48,871 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-11-25T03:37:48,871 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-25T03:37:48,871 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-25T03:37:48,871 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3542 2024-11-25T03:37:48,871 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3650 2024-11-25T03:37:48,871 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-25T03:37:48,871 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1582 2024-11-25T03:37:48,871 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4746 rows 2024-11-25T03:37:48,871 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1581 2024-11-25T03:37:48,871 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4741 rows 2024-11-25T03:37:48,871 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T03:37:48,871 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T03:37:48,871 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:48,871 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:48,871 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:37:48,873 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-25T03:37:48,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-25T03:37:48,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-25T03:37:48,877 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732505868876"}]},"ts":"1732505868876"} 2024-11-25T03:37:48,878 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-25T03:37:48,878 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-25T03:37:48,879 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T03:37:48,882 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e284372599441cd748eba8961521ba1, UNASSIGN}] 2024-11-25T03:37:48,883 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e284372599441cd748eba8961521ba1, UNASSIGN 2024-11-25T03:37:48,885 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=9e284372599441cd748eba8961521ba1, regionState=CLOSING, regionLocation=38a5d2077268,34881,1732505806588 2024-11-25T03:37:48,886 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e284372599441cd748eba8961521ba1, UNASSIGN because future has completed 2024-11-25T03:37:48,887 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T03:37:48,887 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588}] 2024-11-25T03:37:48,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-25T03:37:49,040 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(122): Close 9e284372599441cd748eba8961521ba1 2024-11-25T03:37:49,040 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T03:37:49,040 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1722): Closing 9e284372599441cd748eba8961521ba1, disabling compactions & flushes 2024-11-25T03:37:49,040 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:49,040 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:49,040 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. after waiting 0 ms 2024-11-25T03:37:49,040 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:49,040 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(2902): Flushing 9e284372599441cd748eba8961521ba1 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-25T03:37:49,159 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125242e71a47685432b9357de7bfe65ea40_9e284372599441cd748eba8961521ba1 is 50, key is test_row_0/A:col10/1732505862398/Put/seqid=0 2024-11-25T03:37:49,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-25T03:37:49,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742066_1242 (size=12454) 2024-11-25T03:37:49,234 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:37:49,241 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125242e71a47685432b9357de7bfe65ea40_9e284372599441cd748eba8961521ba1 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125242e71a47685432b9357de7bfe65ea40_9e284372599441cd748eba8961521ba1 2024-11-25T03:37:49,242 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/3d37d79527ad461dac4a0184c9968ee7, store: [table=TestAcidGuarantees family=A region=9e284372599441cd748eba8961521ba1] 2024-11-25T03:37:49,243 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/3d37d79527ad461dac4a0184c9968ee7 is 175, key is test_row_0/A:col10/1732505862398/Put/seqid=0 2024-11-25T03:37:49,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742067_1243 (size=31255) 2024-11-25T03:37:49,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-25T03:37:49,903 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=380, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/3d37d79527ad461dac4a0184c9968ee7 2024-11-25T03:37:49,973 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/be58f0322eea4b79b9b51c8b542d0192 is 50, key is test_row_0/B:col10/1732505862398/Put/seqid=0 2024-11-25T03:37:50,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-25T03:37:50,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742068_1244 (size=12301) 2024-11-25T03:37:50,138 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/be58f0322eea4b79b9b51c8b542d0192 2024-11-25T03:37:50,155 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/f598ea7a52604de6b58b1d8def6f2985 is 50, key is test_row_0/C:col10/1732505862398/Put/seqid=0 2024-11-25T03:37:50,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742069_1245 (size=12301) 2024-11-25T03:37:50,394 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/f598ea7a52604de6b58b1d8def6f2985 2024-11-25T03:37:50,475 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/A/3d37d79527ad461dac4a0184c9968ee7 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/3d37d79527ad461dac4a0184c9968ee7 2024-11-25T03:37:50,508 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/3d37d79527ad461dac4a0184c9968ee7, entries=150, sequenceid=380, filesize=30.5 K 2024-11-25T03:37:50,509 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/B/be58f0322eea4b79b9b51c8b542d0192 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/be58f0322eea4b79b9b51c8b542d0192 2024-11-25T03:37:50,532 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/be58f0322eea4b79b9b51c8b542d0192, entries=150, sequenceid=380, filesize=12.0 K 2024-11-25T03:37:50,553 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/.tmp/C/f598ea7a52604de6b58b1d8def6f2985 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/f598ea7a52604de6b58b1d8def6f2985 2024-11-25T03:37:50,588 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/f598ea7a52604de6b58b1d8def6f2985, entries=150, sequenceid=380, filesize=12.0 K 2024-11-25T03:37:50,591 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=0 B/0 for 9e284372599441cd748eba8961521ba1 in 1551ms, sequenceid=380, compaction requested=false 2024-11-25T03:37:50,738 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/5b65eee8537f48e585855082a70a8cca, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/2bb0e3d1f8ba49b7a4c1ee22369de4eb, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/e88e735532e64424913b93dd5baf1730, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/e9f2be7b55f94605a4e4bd7f4a76296e, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/353578fa21dc4a9cb549ae5e0108826f, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/cd605cec45e94ae089fb134d32ee4fc1, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/9d29a4d1d1f04166b8aec4f11a8ff5d8, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/0d3888a58d61480893e73268e3a745d2, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/42743985edd94a6c9243bad2b63af770, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/469bb20562984f43a0f460f212ae86f3, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/f05491e09c4c4c1283029d4b9f8e8943, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/f79146d967b6470a975c6940ecb1b894, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/8dc28716bc184d408476d21409d399a6, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/72b08dfa6931432aa28e67ca5ff2927b, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/25185d89ded74a6abfe8d4af149cce41, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/2d6a767e83ad4ea59d7f8d8bee026b52, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/8431d0af668b4289927731f22e0ad62e, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/e7d70ce9033e4df6a0d165677873320b, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/fbb7c6b6ab324512aa717638fb9152f8, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/8d75106762534279ba059fc686bfb893, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/6e1809bb71364a13820e7027fb418a9c, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/e06f57f3b0a34de49f4c6c0879cf7265, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/cebcb790eaf8497fa86cf2798a5292ee, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/d74739bf27334d3894e0cc514dddecb9, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/4a98f6fb93c04a80aa8cd1f6d7873f37] to archive 2024-11-25T03:37:50,753 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T03:37:50,757 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/5b65eee8537f48e585855082a70a8cca to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/5b65eee8537f48e585855082a70a8cca 2024-11-25T03:37:50,760 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/2bb0e3d1f8ba49b7a4c1ee22369de4eb to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/2bb0e3d1f8ba49b7a4c1ee22369de4eb 2024-11-25T03:37:50,780 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/e88e735532e64424913b93dd5baf1730 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/e88e735532e64424913b93dd5baf1730 2024-11-25T03:37:50,782 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/e9f2be7b55f94605a4e4bd7f4a76296e to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/e9f2be7b55f94605a4e4bd7f4a76296e 2024-11-25T03:37:50,784 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/353578fa21dc4a9cb549ae5e0108826f to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/353578fa21dc4a9cb549ae5e0108826f 2024-11-25T03:37:50,786 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/cd605cec45e94ae089fb134d32ee4fc1 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/cd605cec45e94ae089fb134d32ee4fc1 2024-11-25T03:37:50,788 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/9d29a4d1d1f04166b8aec4f11a8ff5d8 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/9d29a4d1d1f04166b8aec4f11a8ff5d8 2024-11-25T03:37:50,792 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/0d3888a58d61480893e73268e3a745d2 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/0d3888a58d61480893e73268e3a745d2 2024-11-25T03:37:50,802 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/42743985edd94a6c9243bad2b63af770 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/42743985edd94a6c9243bad2b63af770 2024-11-25T03:37:50,803 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/469bb20562984f43a0f460f212ae86f3 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/469bb20562984f43a0f460f212ae86f3 2024-11-25T03:37:50,805 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/f05491e09c4c4c1283029d4b9f8e8943 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/f05491e09c4c4c1283029d4b9f8e8943 2024-11-25T03:37:50,806 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/f79146d967b6470a975c6940ecb1b894 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/f79146d967b6470a975c6940ecb1b894 2024-11-25T03:37:50,808 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/8dc28716bc184d408476d21409d399a6 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/8dc28716bc184d408476d21409d399a6 2024-11-25T03:37:50,809 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/72b08dfa6931432aa28e67ca5ff2927b to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/72b08dfa6931432aa28e67ca5ff2927b 2024-11-25T03:37:50,810 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/25185d89ded74a6abfe8d4af149cce41 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/25185d89ded74a6abfe8d4af149cce41 2024-11-25T03:37:50,812 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/2d6a767e83ad4ea59d7f8d8bee026b52 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/2d6a767e83ad4ea59d7f8d8bee026b52 2024-11-25T03:37:50,870 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/8431d0af668b4289927731f22e0ad62e to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/8431d0af668b4289927731f22e0ad62e 2024-11-25T03:37:50,872 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/e7d70ce9033e4df6a0d165677873320b to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/e7d70ce9033e4df6a0d165677873320b 2024-11-25T03:37:50,873 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/fbb7c6b6ab324512aa717638fb9152f8 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/fbb7c6b6ab324512aa717638fb9152f8 2024-11-25T03:37:50,875 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/8d75106762534279ba059fc686bfb893 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/8d75106762534279ba059fc686bfb893 2024-11-25T03:37:50,877 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/6e1809bb71364a13820e7027fb418a9c to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/6e1809bb71364a13820e7027fb418a9c 2024-11-25T03:37:50,878 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/e06f57f3b0a34de49f4c6c0879cf7265 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/e06f57f3b0a34de49f4c6c0879cf7265 2024-11-25T03:37:50,880 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/cebcb790eaf8497fa86cf2798a5292ee to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/cebcb790eaf8497fa86cf2798a5292ee 2024-11-25T03:37:50,908 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/d74739bf27334d3894e0cc514dddecb9 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/d74739bf27334d3894e0cc514dddecb9 2024-11-25T03:37:50,909 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/4a98f6fb93c04a80aa8cd1f6d7873f37 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/4a98f6fb93c04a80aa8cd1f6d7873f37 2024-11-25T03:37:50,949 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/1c872f79dc9843d8bc8b160f29350b47, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/e4c061a8a8dd4284af6ed27de860d8cb, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/e71896520f18492fb2c734866bbb4e6c, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/2c02bc8164c44021a1a6aded80c73958, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/2357accf81f04b9da6399c8a80f88e94, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/7584232a4e154410b2e3fd1b6ca4bbf0, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/ab558a8ba57846bcad4896923a6392fc, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/686db1342ff94a9bbf27d3aec0877107, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/0e3f68dde8454979bdaff5d5cbda1924, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/dd60528f421b4b6a981ad60ee64f5189, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/a39599753f644afe99f11efa81b0bda7, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/527decd5ff1f4174b80817cf5f2df565, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/5cbf16466bb84c0296892deebc5530e5, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/2fa3fade0341434c953910057e73f310, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/2c19961d7bd3438d96bd08f57b64f101, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/73e15e802b214d4b98980af2a6e2a5cc, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/1ef5aea670304b458094730a7cbd0003, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/4305116cd2e040b2ac475647a32db1dd, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/79a155449d0a448fb814392c92f71af5, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/f0068b96da21451aa7bc75bee80a2530, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/d5a7586ee71c435b9755e2b67cf58f58, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/ac9fc295181c4badaf59a7313283fbed, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/aad98ab33d754328ae4f693263a5d6a8, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/9985f7442e7b402db900fa2fbc8b7edb, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/366af6ef0b22402bbd0940bb3d684b68] to archive 2024-11-25T03:37:50,951 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T03:37:50,953 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/1c872f79dc9843d8bc8b160f29350b47 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/1c872f79dc9843d8bc8b160f29350b47 2024-11-25T03:37:50,955 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/e4c061a8a8dd4284af6ed27de860d8cb to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/e4c061a8a8dd4284af6ed27de860d8cb 2024-11-25T03:37:50,957 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/e71896520f18492fb2c734866bbb4e6c to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/e71896520f18492fb2c734866bbb4e6c 2024-11-25T03:37:50,961 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/2c02bc8164c44021a1a6aded80c73958 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/2c02bc8164c44021a1a6aded80c73958 2024-11-25T03:37:50,962 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/2357accf81f04b9da6399c8a80f88e94 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/2357accf81f04b9da6399c8a80f88e94 2024-11-25T03:37:50,989 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/7584232a4e154410b2e3fd1b6ca4bbf0 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/7584232a4e154410b2e3fd1b6ca4bbf0 2024-11-25T03:37:51,020 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/ab558a8ba57846bcad4896923a6392fc to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/ab558a8ba57846bcad4896923a6392fc 2024-11-25T03:37:51,022 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/686db1342ff94a9bbf27d3aec0877107 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/686db1342ff94a9bbf27d3aec0877107 2024-11-25T03:37:51,023 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/0e3f68dde8454979bdaff5d5cbda1924 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/0e3f68dde8454979bdaff5d5cbda1924 2024-11-25T03:37:51,027 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/dd60528f421b4b6a981ad60ee64f5189 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/dd60528f421b4b6a981ad60ee64f5189 2024-11-25T03:37:51,031 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/a39599753f644afe99f11efa81b0bda7 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/a39599753f644afe99f11efa81b0bda7 2024-11-25T03:37:51,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-25T03:37:51,067 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/527decd5ff1f4174b80817cf5f2df565 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/527decd5ff1f4174b80817cf5f2df565 2024-11-25T03:37:51,104 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/5cbf16466bb84c0296892deebc5530e5 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/5cbf16466bb84c0296892deebc5530e5 2024-11-25T03:37:51,111 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/2fa3fade0341434c953910057e73f310 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/2fa3fade0341434c953910057e73f310 2024-11-25T03:37:51,113 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/2c19961d7bd3438d96bd08f57b64f101 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/2c19961d7bd3438d96bd08f57b64f101 2024-11-25T03:37:51,114 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/73e15e802b214d4b98980af2a6e2a5cc to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/73e15e802b214d4b98980af2a6e2a5cc 2024-11-25T03:37:51,115 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/1ef5aea670304b458094730a7cbd0003 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/1ef5aea670304b458094730a7cbd0003 2024-11-25T03:37:51,117 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/4305116cd2e040b2ac475647a32db1dd to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/4305116cd2e040b2ac475647a32db1dd 2024-11-25T03:37:51,118 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/79a155449d0a448fb814392c92f71af5 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/79a155449d0a448fb814392c92f71af5 2024-11-25T03:37:51,120 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/f0068b96da21451aa7bc75bee80a2530 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/f0068b96da21451aa7bc75bee80a2530 2024-11-25T03:37:51,123 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/d5a7586ee71c435b9755e2b67cf58f58 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/d5a7586ee71c435b9755e2b67cf58f58 2024-11-25T03:37:51,124 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/ac9fc295181c4badaf59a7313283fbed to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/ac9fc295181c4badaf59a7313283fbed 2024-11-25T03:37:51,125 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/aad98ab33d754328ae4f693263a5d6a8 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/aad98ab33d754328ae4f693263a5d6a8 2024-11-25T03:37:51,126 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/9985f7442e7b402db900fa2fbc8b7edb to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/9985f7442e7b402db900fa2fbc8b7edb 2024-11-25T03:37:51,127 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/366af6ef0b22402bbd0940bb3d684b68 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/366af6ef0b22402bbd0940bb3d684b68 2024-11-25T03:37:51,300 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/d0df032432bc4dd0925f85a5913c5295, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/c2e1e591785846e58646371118bbc2ae, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/24f796b495bb410580e242aea444b720, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/178ae2e30f164492a5bcea1cd13092cc, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/aebda93c62b242a7a3fb8b403f59f0d0, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/726121ed23b44824ba99dae8b5d687f0, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/0e4ae549bf65460ba3f50c694c4c2a93, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/de24325ce53249b197718ba311dcddae, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/2d9b58e0c5dc4a699c6df7a471d6742d, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/5333c98762ac477598ca290db53bc2b9, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/6dab53fa845a4d489cce677f6b07425a, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/0b6bbb30e28d433cb212b4ab000205ef, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/e1327af6e0644ae09c114c28c5ecc1d2, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/cfcf0bba89f14c3b979e3115b0b1b629, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/c105947ef05a4ab3b1731c6d2e9faa7d, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/83e7010886e1424cb8bff457b59a8117, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/711c92af6eee455d8c4fe512ce5b3cdb, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/1dd58621bd6049c8befd9fd249a2fa45, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/4121132e26634f91b1f42364eae4723a, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/44faa06b679743e49da24c15bce5c869, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/ccf7532908b648aa8220cbc254e33e65, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/6c043195829c490da48352c136d350fb, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/e078e2129a0d4b9bbc8e85629cb42d10, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/42a915331a134593953a4df1473e60dd, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/473a1f658ca8429d99ac4fa21457f84d] to archive 2024-11-25T03:37:51,302 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T03:37:51,304 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/d0df032432bc4dd0925f85a5913c5295 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/d0df032432bc4dd0925f85a5913c5295 2024-11-25T03:37:51,307 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/c2e1e591785846e58646371118bbc2ae to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/c2e1e591785846e58646371118bbc2ae 2024-11-25T03:37:51,315 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/24f796b495bb410580e242aea444b720 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/24f796b495bb410580e242aea444b720 2024-11-25T03:37:51,317 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/178ae2e30f164492a5bcea1cd13092cc to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/178ae2e30f164492a5bcea1cd13092cc 2024-11-25T03:37:51,318 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/aebda93c62b242a7a3fb8b403f59f0d0 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/aebda93c62b242a7a3fb8b403f59f0d0 2024-11-25T03:37:51,320 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/726121ed23b44824ba99dae8b5d687f0 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/726121ed23b44824ba99dae8b5d687f0 2024-11-25T03:37:51,321 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/0e4ae549bf65460ba3f50c694c4c2a93 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/0e4ae549bf65460ba3f50c694c4c2a93 2024-11-25T03:37:51,322 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/de24325ce53249b197718ba311dcddae to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/de24325ce53249b197718ba311dcddae 2024-11-25T03:37:51,324 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/2d9b58e0c5dc4a699c6df7a471d6742d to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/2d9b58e0c5dc4a699c6df7a471d6742d 2024-11-25T03:37:51,349 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/5333c98762ac477598ca290db53bc2b9 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/5333c98762ac477598ca290db53bc2b9 2024-11-25T03:37:51,357 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/6dab53fa845a4d489cce677f6b07425a to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/6dab53fa845a4d489cce677f6b07425a 2024-11-25T03:37:51,358 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/0b6bbb30e28d433cb212b4ab000205ef to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/0b6bbb30e28d433cb212b4ab000205ef 2024-11-25T03:37:51,360 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/e1327af6e0644ae09c114c28c5ecc1d2 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/e1327af6e0644ae09c114c28c5ecc1d2 2024-11-25T03:37:51,361 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/cfcf0bba89f14c3b979e3115b0b1b629 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/cfcf0bba89f14c3b979e3115b0b1b629 2024-11-25T03:37:51,363 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/c105947ef05a4ab3b1731c6d2e9faa7d to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/c105947ef05a4ab3b1731c6d2e9faa7d 2024-11-25T03:37:51,364 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/83e7010886e1424cb8bff457b59a8117 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/83e7010886e1424cb8bff457b59a8117 2024-11-25T03:37:51,365 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/711c92af6eee455d8c4fe512ce5b3cdb to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/711c92af6eee455d8c4fe512ce5b3cdb 2024-11-25T03:37:51,367 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/1dd58621bd6049c8befd9fd249a2fa45 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/1dd58621bd6049c8befd9fd249a2fa45 2024-11-25T03:37:51,371 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/4121132e26634f91b1f42364eae4723a to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/4121132e26634f91b1f42364eae4723a 2024-11-25T03:37:51,373 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/44faa06b679743e49da24c15bce5c869 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/44faa06b679743e49da24c15bce5c869 2024-11-25T03:37:51,374 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/ccf7532908b648aa8220cbc254e33e65 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/ccf7532908b648aa8220cbc254e33e65 2024-11-25T03:37:51,375 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/6c043195829c490da48352c136d350fb to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/6c043195829c490da48352c136d350fb 2024-11-25T03:37:51,377 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/e078e2129a0d4b9bbc8e85629cb42d10 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/e078e2129a0d4b9bbc8e85629cb42d10 2024-11-25T03:37:51,387 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/42a915331a134593953a4df1473e60dd to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/42a915331a134593953a4df1473e60dd 2024-11-25T03:37:51,389 DEBUG [StoreCloser-TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/473a1f658ca8429d99ac4fa21457f84d to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/473a1f658ca8429d99ac4fa21457f84d 2024-11-25T03:37:51,434 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/recovered.edits/383.seqid, newMaxSeqId=383, maxSeqId=4 2024-11-25T03:37:51,435 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. 2024-11-25T03:37:51,436 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1676): Region close journal for 9e284372599441cd748eba8961521ba1: Waiting for close lock at 1732505869040Running coprocessor pre-close hooks at 1732505869040Disabling compacts and flushes for region at 1732505869040Disabling writes for close at 1732505869040Obtaining lock to block concurrent updates at 1732505869040Preparing flush snapshotting stores in 9e284372599441cd748eba8961521ba1 at 1732505869040Finished memstore snapshotting TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1., syncing WAL and waiting on mvcc, flushsize=dataSize=48090, getHeapSize=126720, getOffHeapSize=0, getCellsCount=1050 at 1732505869041 (+1 ms)Flushing stores of TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1. at 1732505869042 (+1 ms)Flushing 9e284372599441cd748eba8961521ba1/A: creating writer at 1732505869042Flushing mob file 9e284372599441cd748eba8961521ba1/A: appending metadata at 1732505869158 (+116 ms)Flushing mob file 9e284372599441cd748eba8961521ba1/A: closing flushed file at 1732505869158Flushing 9e284372599441cd748eba8961521ba1/A: appending metadata at 1732505869242 (+84 ms)Flushing 9e284372599441cd748eba8961521ba1/A: closing flushed file at 1732505869242Flushing 9e284372599441cd748eba8961521ba1/B: creating writer at 1732505869909 (+667 ms)Flushing 9e284372599441cd748eba8961521ba1/B: appending metadata at 1732505869972 (+63 ms)Flushing 9e284372599441cd748eba8961521ba1/B: closing flushed file at 1732505869972Flushing 9e284372599441cd748eba8961521ba1/C: creating writer at 1732505870144 (+172 ms)Flushing 9e284372599441cd748eba8961521ba1/C: appending metadata at 1732505870155 (+11 ms)Flushing 9e284372599441cd748eba8961521ba1/C: closing flushed file at 1732505870155Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@32941dcf: reopening flushed file at 1732505870473 (+318 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c604310: reopening flushed file at 1732505870508 (+35 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c8458d1: reopening flushed file at 1732505870532 (+24 ms)Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=0 B/0 for 9e284372599441cd748eba8961521ba1 in 1551ms, sequenceid=380, compaction requested=false at 1732505870591 (+59 ms)Writing region close event to WAL at 1732505871428 (+837 ms)Running coprocessor post-close hooks at 1732505871435 (+7 ms)Closed at 1732505871435 2024-11-25T03:37:51,439 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=9e284372599441cd748eba8961521ba1, regionState=CLOSED 2024-11-25T03:37:51,440 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(157): Closed 9e284372599441cd748eba8961521ba1 2024-11-25T03:37:51,441 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 because future has completed 2024-11-25T03:37:51,444 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-25T03:37:51,444 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; CloseRegionProcedure 9e284372599441cd748eba8961521ba1, server=38a5d2077268,34881,1732505806588 in 2.5550 sec 2024-11-25T03:37:51,446 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-25T03:37:51,446 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e284372599441cd748eba8961521ba1, UNASSIGN in 2.5620 sec 2024-11-25T03:37:51,451 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732505871451"}]},"ts":"1732505871451"} 2024-11-25T03:37:51,452 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-25T03:37:51,452 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-25T03:37:51,454 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.5800 sec 2024-11-25T03:37:51,454 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-25T03:37:51,454 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.5680 sec 2024-11-25T03:37:53,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-25T03:37:53,058 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:37:53,058 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-25T03:37:53,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:37:53,061 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=64, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:37:53,061 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=64, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:37:53,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-25T03:37:53,104 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1 2024-11-25T03:37:53,125 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A, FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B, FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C, FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/recovered.edits] 2024-11-25T03:37:53,129 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/3d37d79527ad461dac4a0184c9968ee7 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/3d37d79527ad461dac4a0184c9968ee7 2024-11-25T03:37:53,153 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/6f889c4fb9d146e9b551e01c3e15c4dc to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/A/6f889c4fb9d146e9b551e01c3e15c4dc 2024-11-25T03:37:53,165 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/0dd0135b9c3744ddafefaff6e0ab2efa to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/0dd0135b9c3744ddafefaff6e0ab2efa 2024-11-25T03:37:53,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-25T03:37:53,189 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/be58f0322eea4b79b9b51c8b542d0192 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/B/be58f0322eea4b79b9b51c8b542d0192 2024-11-25T03:37:53,193 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/df55f2b7d2fa40afb33206a841c44988 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/df55f2b7d2fa40afb33206a841c44988 2024-11-25T03:37:53,195 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/f598ea7a52604de6b58b1d8def6f2985 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/C/f598ea7a52604de6b58b1d8def6f2985 2024-11-25T03:37:53,198 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/recovered.edits/383.seqid to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1/recovered.edits/383.seqid 2024-11-25T03:37:53,199 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/9e284372599441cd748eba8961521ba1 2024-11-25T03:37:53,208 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-25T03:37:53,212 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-25T03:37:53,215 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-25T03:37:53,244 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411250a63468da2be49868fe995815cc379a3_9e284372599441cd748eba8961521ba1 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411250a63468da2be49868fe995815cc379a3_9e284372599441cd748eba8961521ba1 2024-11-25T03:37:53,246 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112518145d7828a04a25be2fad49e076c89d_9e284372599441cd748eba8961521ba1 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112518145d7828a04a25be2fad49e076c89d_9e284372599441cd748eba8961521ba1 2024-11-25T03:37:53,247 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125242e71a47685432b9357de7bfe65ea40_9e284372599441cd748eba8961521ba1 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125242e71a47685432b9357de7bfe65ea40_9e284372599441cd748eba8961521ba1 2024-11-25T03:37:53,249 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411252f97ae056d144344a371f51aa95ba6d1_9e284372599441cd748eba8961521ba1 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411252f97ae056d144344a371f51aa95ba6d1_9e284372599441cd748eba8961521ba1 2024-11-25T03:37:53,250 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112559235a56703f490f9053a3879a2b7598_9e284372599441cd748eba8961521ba1 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112559235a56703f490f9053a3879a2b7598_9e284372599441cd748eba8961521ba1 2024-11-25T03:37:53,252 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411255ec17bc38d0242c3bb0bb0b918920265_9e284372599441cd748eba8961521ba1 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411255ec17bc38d0242c3bb0bb0b918920265_9e284372599441cd748eba8961521ba1 2024-11-25T03:37:53,253 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411256b5f066e9ef54681aa7c96dfabb8f846_9e284372599441cd748eba8961521ba1 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411256b5f066e9ef54681aa7c96dfabb8f846_9e284372599441cd748eba8961521ba1 2024-11-25T03:37:53,255 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411256be95ae7b7a1415393a9ad25e25ed3a3_9e284372599441cd748eba8961521ba1 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411256be95ae7b7a1415393a9ad25e25ed3a3_9e284372599441cd748eba8961521ba1 2024-11-25T03:37:53,256 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112573d84be70abd49f58705bcc55592bebe_9e284372599441cd748eba8961521ba1 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112573d84be70abd49f58705bcc55592bebe_9e284372599441cd748eba8961521ba1 2024-11-25T03:37:53,259 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125801d7b90fd0241eab153492c288934f7_9e284372599441cd748eba8961521ba1 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125801d7b90fd0241eab153492c288934f7_9e284372599441cd748eba8961521ba1 2024-11-25T03:37:53,260 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125827d139107004cf39586cb5ed7284e94_9e284372599441cd748eba8961521ba1 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125827d139107004cf39586cb5ed7284e94_9e284372599441cd748eba8961521ba1 2024-11-25T03:37:53,262 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125c467437032794a2f8863def8d8eb1ef2_9e284372599441cd748eba8961521ba1 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125c467437032794a2f8863def8d8eb1ef2_9e284372599441cd748eba8961521ba1 2024-11-25T03:37:53,263 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125c511b59c956248dfa78378afbd3e1404_9e284372599441cd748eba8961521ba1 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125c511b59c956248dfa78378afbd3e1404_9e284372599441cd748eba8961521ba1 2024-11-25T03:37:53,264 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125cf664b680e814ce7800349afda548565_9e284372599441cd748eba8961521ba1 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125cf664b680e814ce7800349afda548565_9e284372599441cd748eba8961521ba1 2024-11-25T03:37:53,266 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125d283d5fe32374436a85f2e46de100dba_9e284372599441cd748eba8961521ba1 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125d283d5fe32374436a85f2e46de100dba_9e284372599441cd748eba8961521ba1 2024-11-25T03:37:53,267 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125d50fc6308edd4f60bf2cf3f7f5e879c5_9e284372599441cd748eba8961521ba1 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125d50fc6308edd4f60bf2cf3f7f5e879c5_9e284372599441cd748eba8961521ba1 2024-11-25T03:37:53,272 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125de3b812063d04d3ea8a25ac82985eabe_9e284372599441cd748eba8961521ba1 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125de3b812063d04d3ea8a25ac82985eabe_9e284372599441cd748eba8961521ba1 2024-11-25T03:37:53,277 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125ed8433757f864a9f85d88e6b32dee154_9e284372599441cd748eba8961521ba1 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125ed8433757f864a9f85d88e6b32dee154_9e284372599441cd748eba8961521ba1 2024-11-25T03:37:53,278 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125f13bd85367c645d9bb506b75444f2660_9e284372599441cd748eba8961521ba1 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125f13bd85367c645d9bb506b75444f2660_9e284372599441cd748eba8961521ba1 2024-11-25T03:37:53,279 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-25T03:37:53,283 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=64, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:37:53,291 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-25T03:37:53,295 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-25T03:37:53,305 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=64, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:37:53,305 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-25T03:37:53,305 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732505873305"}]},"ts":"9223372036854775807"} 2024-11-25T03:37:53,322 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-25T03:37:53,322 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 9e284372599441cd748eba8961521ba1, NAME => 'TestAcidGuarantees,,1732505837221.9e284372599441cd748eba8961521ba1.', STARTKEY => '', ENDKEY => ''}] 2024-11-25T03:37:53,322 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-25T03:37:53,322 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732505873322"}]},"ts":"9223372036854775807"} 2024-11-25T03:37:53,325 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-25T03:37:53,326 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=64, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:37:53,342 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 268 msec 2024-11-25T03:37:53,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-25T03:37:53,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-25T03:37:53,378 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:37:53,450 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=238 (was 239), OpenFileDescriptor=445 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=608 (was 348) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5961 (was 6073) 2024-11-25T03:37:53,650 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=238, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=935, ProcessCount=11, AvailableMemoryMB=5960 2024-11-25T03:37:53,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T03:37:53,652 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T03:37:53,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=65, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-25T03:37:53,655 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T03:37:53,655 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:37:53,656 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T03:37:53,656 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 65 2024-11-25T03:37:53,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-25T03:37:53,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-25T03:37:53,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742070_1246 (size=959) 2024-11-25T03:37:53,876 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3 2024-11-25T03:37:53,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-25T03:37:54,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742071_1247 (size=53) 2024-11-25T03:37:54,087 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:37:54,087 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c6d2702d0a1f8c4a0fdcb6498eb058d5, disabling compactions & flushes 2024-11-25T03:37:54,087 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:37:54,087 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:37:54,087 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. after waiting 0 ms 2024-11-25T03:37:54,087 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:37:54,087 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:37:54,087 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c6d2702d0a1f8c4a0fdcb6498eb058d5: Waiting for close lock at 1732505874087Disabling compacts and flushes for region at 1732505874087Disabling writes for close at 1732505874087Writing region close event to WAL at 1732505874087Closed at 1732505874087 2024-11-25T03:37:54,096 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T03:37:54,097 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732505874096"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732505874096"}]},"ts":"1732505874096"} 2024-11-25T03:37:54,099 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T03:37:54,100 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T03:37:54,100 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732505874100"}]},"ts":"1732505874100"} 2024-11-25T03:37:54,103 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-25T03:37:54,103 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c6d2702d0a1f8c4a0fdcb6498eb058d5, ASSIGN}] 2024-11-25T03:37:54,105 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c6d2702d0a1f8c4a0fdcb6498eb058d5, ASSIGN 2024-11-25T03:37:54,106 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c6d2702d0a1f8c4a0fdcb6498eb058d5, ASSIGN; state=OFFLINE, location=38a5d2077268,34881,1732505806588; forceNewPlan=false, retain=false 2024-11-25T03:37:54,257 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=c6d2702d0a1f8c4a0fdcb6498eb058d5, regionState=OPENING, regionLocation=38a5d2077268,34881,1732505806588 2024-11-25T03:37:54,259 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c6d2702d0a1f8c4a0fdcb6498eb058d5, ASSIGN because future has completed 2024-11-25T03:37:54,259 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588}] 2024-11-25T03:37:54,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-25T03:37:54,471 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:37:54,471 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7752): Opening region: {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:37:54,471 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c6d2702d0a1f8c4a0fdcb6498eb058d5 2024-11-25T03:37:54,472 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:37:54,472 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7794): checking encryption for c6d2702d0a1f8c4a0fdcb6498eb058d5 2024-11-25T03:37:54,472 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7797): checking classloading for c6d2702d0a1f8c4a0fdcb6498eb058d5 2024-11-25T03:37:54,484 INFO [StoreOpener-c6d2702d0a1f8c4a0fdcb6498eb058d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c6d2702d0a1f8c4a0fdcb6498eb058d5 2024-11-25T03:37:54,501 INFO [StoreOpener-c6d2702d0a1f8c4a0fdcb6498eb058d5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c6d2702d0a1f8c4a0fdcb6498eb058d5 columnFamilyName A 2024-11-25T03:37:54,501 DEBUG [StoreOpener-c6d2702d0a1f8c4a0fdcb6498eb058d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:37:54,513 INFO [StoreOpener-c6d2702d0a1f8c4a0fdcb6498eb058d5-1 {}] regionserver.HStore(327): Store=c6d2702d0a1f8c4a0fdcb6498eb058d5/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:37:54,513 INFO [StoreOpener-c6d2702d0a1f8c4a0fdcb6498eb058d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c6d2702d0a1f8c4a0fdcb6498eb058d5 2024-11-25T03:37:54,515 INFO [StoreOpener-c6d2702d0a1f8c4a0fdcb6498eb058d5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c6d2702d0a1f8c4a0fdcb6498eb058d5 columnFamilyName B 2024-11-25T03:37:54,515 DEBUG [StoreOpener-c6d2702d0a1f8c4a0fdcb6498eb058d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:37:54,516 INFO [StoreOpener-c6d2702d0a1f8c4a0fdcb6498eb058d5-1 {}] regionserver.HStore(327): Store=c6d2702d0a1f8c4a0fdcb6498eb058d5/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:37:54,516 INFO [StoreOpener-c6d2702d0a1f8c4a0fdcb6498eb058d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c6d2702d0a1f8c4a0fdcb6498eb058d5 2024-11-25T03:37:54,527 INFO [StoreOpener-c6d2702d0a1f8c4a0fdcb6498eb058d5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c6d2702d0a1f8c4a0fdcb6498eb058d5 columnFamilyName C 2024-11-25T03:37:54,527 DEBUG [StoreOpener-c6d2702d0a1f8c4a0fdcb6498eb058d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:37:54,528 INFO [StoreOpener-c6d2702d0a1f8c4a0fdcb6498eb058d5-1 {}] regionserver.HStore(327): Store=c6d2702d0a1f8c4a0fdcb6498eb058d5/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:37:54,528 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1038): replaying wal for c6d2702d0a1f8c4a0fdcb6498eb058d5 2024-11-25T03:37:54,529 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5 2024-11-25T03:37:54,530 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5 2024-11-25T03:37:54,539 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1048): stopping wal replay for c6d2702d0a1f8c4a0fdcb6498eb058d5 2024-11-25T03:37:54,539 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1060): Cleaning up temporary data for c6d2702d0a1f8c4a0fdcb6498eb058d5 2024-11-25T03:37:54,540 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T03:37:54,543 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1093): writing seq id for c6d2702d0a1f8c4a0fdcb6498eb058d5 2024-11-25T03:37:54,592 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:37:54,604 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1114): Opened c6d2702d0a1f8c4a0fdcb6498eb058d5; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75307758, jitterRate=0.12217304110527039}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T03:37:54,605 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c6d2702d0a1f8c4a0fdcb6498eb058d5 2024-11-25T03:37:54,605 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1006): Region open journal for c6d2702d0a1f8c4a0fdcb6498eb058d5: Running coprocessor pre-open hook at 1732505874472Writing region info on filesystem at 1732505874472Initializing all the Stores at 1732505874473 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732505874473Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732505874484 (+11 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732505874484Cleaning up temporary data from old regions at 1732505874539 (+55 ms)Running coprocessor post-open hooks at 1732505874605 (+66 ms)Region opened successfully at 1732505874605 2024-11-25T03:37:54,607 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., pid=67, masterSystemTime=1732505874416 2024-11-25T03:37:54,611 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=c6d2702d0a1f8c4a0fdcb6498eb058d5, regionState=OPEN, openSeqNum=2, regionLocation=38a5d2077268,34881,1732505806588 2024-11-25T03:37:54,612 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:37:54,612 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:37:54,615 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 because future has completed 2024-11-25T03:37:54,618 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-25T03:37:54,618 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 in 357 msec 2024-11-25T03:37:54,624 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-11-25T03:37:54,624 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c6d2702d0a1f8c4a0fdcb6498eb058d5, ASSIGN in 515 msec 2024-11-25T03:37:54,629 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T03:37:54,629 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732505874629"}]},"ts":"1732505874629"} 2024-11-25T03:37:54,630 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-25T03:37:54,631 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T03:37:54,633 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 979 msec 2024-11-25T03:37:54,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-25T03:37:54,798 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:37:54,802 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fe11fca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:54,802 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:37:54,802 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:37:54,805 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:37:54,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:37:54,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:37:54,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@500ea305, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:54,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:37:54,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:37:54,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:54,808 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35240, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:37:54,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28126738, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:54,809 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:37:54,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:37:54,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:37:54,811 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57798, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:37:54,812 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T03:37:54,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 38a5d2077268,42243,1732505805780 2024-11-25T03:37:54,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2e956dfa 2024-11-25T03:37:54,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T03:37:54,817 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35252, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T03:37:54,828 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34b5b273, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:54,829 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:37:54,829 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:37:54,829 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:37:54,830 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:37:54,830 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:37:54,830 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d74817c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:54,830 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:37:54,830 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:37:54,830 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:54,831 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35268, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:37:54,832 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6017ec17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:54,833 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1240be3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:54,833 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:37:54,833 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:37:54,834 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:37:54,834 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:37:54,834 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:37:54,834 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26e3c744, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:54,834 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:37:54,834 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:37:54,835 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:54,836 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35290, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:37:54,837 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25fa6d59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:54,838 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2885eb44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:54,838 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:37:54,838 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:37:54,839 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:37:54,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:37:54,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:37:54,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fdd7f21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:54,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:37:54,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:37:54,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:54,841 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35306, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:37:54,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55e296cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:54,889 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692a6227, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:54,889 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:37:54,889 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:37:54,890 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:37:54,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:37:54,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:37:54,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@140dfd06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:54,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:37:54,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:37:54,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:54,893 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35324, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:37:54,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39c61960, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:54,900 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@128d97bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:54,900 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:37:54,901 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:37:54,910 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:37:54,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:37:54,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:37:54,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5de251f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:54,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:37:54,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:37:54,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:54,925 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35350, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:37:54,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fee3ae1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:54,956 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@626de388, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:54,957 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:37:54,957 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:37:54,958 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:37:54,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:37:54,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:37:54,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b41a2aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:54,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:37:54,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:37:54,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:54,976 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35368, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:37:54,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d1fd712, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:55,081 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47a5c877, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:55,081 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:37:55,081 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:37:55,087 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:37:55,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:37:55,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:37:55,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a41e3aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:55,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:37:55,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:37:55,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:55,095 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35392, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:37:55,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1299e782, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:55,140 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@679f7426, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:55,141 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:37:55,141 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:37:55,142 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:37:55,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:37:55,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:37:55,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cb4c1f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:55,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:37:55,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:37:55,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:55,145 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35422, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:37:55,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31e99969, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:55,149 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b71af22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:55,149 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:37:55,149 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:37:55,150 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:37:55,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:37:55,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:37:55,151 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@246ec6e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:55,151 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:37:55,151 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:37:55,151 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:55,152 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35448, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:37:55,153 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e3db066, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:55,172 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@728d338, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:55,172 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:37:55,172 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:37:55,173 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:37:55,174 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:37:55,174 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:37:55,174 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a8a4f8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:55,174 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:37:55,174 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:37:55,176 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:37:55,181 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35466, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:37:55,182 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c0eb5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:37:55,309 DEBUG [Thread-1152 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:37:55,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:37:55,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:37:55,312 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:37:55,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2] 2024-11-25T03:37:55,335 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:37:55,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-25T03:37:55,336 DEBUG [Thread-1159 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:37:55,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-25T03:37:55,337 DEBUG [Thread-1161 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:37:55,337 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:37:55,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:37:55,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:37:55,338 DEBUG [Thread-1165 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:37:55,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:37:55,339 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:37:55,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:37:55,339 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:37:55,340 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57820, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:37:55,340 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57824, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:37:55,341 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:37:55,341 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:37:55,345 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57840, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:37:55,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2] 2024-11-25T03:37:55,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2] 2024-11-25T03:37:55,358 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2] 2024-11-25T03:37:55,380 DEBUG [Thread-1148 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:37:55,381 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:37:55,381 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:37:55,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on c6d2702d0a1f8c4a0fdcb6498eb058d5 2024-11-25T03:37:55,388 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:37:55,388 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c6d2702d0a1f8c4a0fdcb6498eb058d5 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-25T03:37:55,389 DEBUG [Thread-1167 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:37:55,389 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57844, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:37:55,390 DEBUG [Thread-1167 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:37:55,390 DEBUG [Thread-1167 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:37:55,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2] 2024-11-25T03:37:55,392 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57850, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:37:55,393 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2] 2024-11-25T03:37:55,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-25T03:37:55,465 DEBUG [Thread-1150 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:37:55,466 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:37:55,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:37:55,469 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57856, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:37:55,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2] 2024-11-25T03:37:55,497 DEBUG [Thread-1156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:37:55,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:37:55,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:37:55,500 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57870, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:37:55,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2] 2024-11-25T03:37:55,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-25T03:37:55,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:37:55,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. as already flushing 2024-11-25T03:37:55,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:37:55,526 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:55,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:55,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:55,530 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42243 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=38a5d2077268,34881,1732505806588, table=TestAcidGuarantees, region=c6d2702d0a1f8c4a0fdcb6498eb058d5. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-25T03:37:55,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-25T03:37:55,682 DEBUG [Thread-1154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:37:55,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-25T03:37:55,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:37:55,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. as already flushing 2024-11-25T03:37:55,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:37:55,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:55,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:55,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:37:55,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:37:55,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:55,688 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57872, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:37:55,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2] 2024-11-25T03:37:55,793 DEBUG [Thread-1163 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:37:55,794 DEBUG [Thread-1163 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:37:55,795 DEBUG [Thread-1163 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:37:55,798 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57886, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:37:55,800 DEBUG [Thread-1163 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2] 2024-11-25T03:37:55,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:55,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57856 deadline: 1732505935766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:55,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:55,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:55,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:55,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/6b4bcc156fcf4c99912a3bbc86f9a131 is 50, key is test_row_0/A:col10/1732505875386/Put/seqid=0 2024-11-25T03:37:55,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:55,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57870 deadline: 1732505935808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:55,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:55,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1732505935825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:55,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:55,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:55,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:55,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:55,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:55,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57844 deadline: 1732505935827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:55,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:55,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:55,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:55,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:55,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:55,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:55,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505935827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:55,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-25T03:37:55,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:55,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:55,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:55,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:37:55,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. as already flushing 2024-11-25T03:37:55,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:37:55,899 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:55,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:55,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:55,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742072_1248 (size=12001) 2024-11-25T03:37:55,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/6b4bcc156fcf4c99912a3bbc86f9a131 2024-11-25T03:37:55,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-25T03:37:56,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:56,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57856 deadline: 1732505935939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:56,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:56,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:56,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:56,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:56,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57870 deadline: 1732505935977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:56,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:56,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:56,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:56,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:56,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57844 deadline: 1732505935977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:56,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:56,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:56,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:56,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:56,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1732505935980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:56,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:56,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505936008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:56,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:56,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:56,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:56,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-25T03:37:56,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:37:56,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. as already flushing 2024-11-25T03:37:56,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:37:56,184 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:56,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:56,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:56,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:56,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57870 deadline: 1732505936257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:56,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:56,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:56,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:56,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:56,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57844 deadline: 1732505936257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:56,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:56,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:56,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:56,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:56,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57856 deadline: 1732505936260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:56,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:56,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:56,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:56,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/ff8674afbcb94968b039ec8b7a6323e9 is 50, key is test_row_0/B:col10/1732505875386/Put/seqid=0 2024-11-25T03:37:56,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:56,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505936400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:56,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:56,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:56,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:56,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-25T03:37:56,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742073_1249 (size=12001) 2024-11-25T03:37:56,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:37:56,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. as already flushing 2024-11-25T03:37:56,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:37:56,413 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:56,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:56,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:56,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/ff8674afbcb94968b039ec8b7a6323e9 2024-11-25T03:37:56,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:56,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1732505936410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:56,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:56,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:56,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:56,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/67a1b3058b724c33bc2e579763a2258b is 50, key is test_row_0/C:col10/1732505875386/Put/seqid=0 2024-11-25T03:37:56,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-25T03:37:56,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-25T03:37:56,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:37:56,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. as already flushing 2024-11-25T03:37:56,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:37:56,644 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:56,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:56,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:56,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:56,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57856 deadline: 1732505936580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:56,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:56,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57844 deadline: 1732505936588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:56,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:56,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:56,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:56,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:56,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57870 deadline: 1732505936588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:56,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:56,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:56,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:56,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:56,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:56,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:56,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742074_1250 (size=12001) 2024-11-25T03:37:56,714 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/67a1b3058b724c33bc2e579763a2258b 2024-11-25T03:37:56,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:56,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505936707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:56,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:56,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:56,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:56,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/6b4bcc156fcf4c99912a3bbc86f9a131 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/6b4bcc156fcf4c99912a3bbc86f9a131 2024-11-25T03:37:56,725 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/6b4bcc156fcf4c99912a3bbc86f9a131, entries=150, sequenceid=12, filesize=11.7 K 2024-11-25T03:37:56,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/ff8674afbcb94968b039ec8b7a6323e9 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/ff8674afbcb94968b039ec8b7a6323e9 2024-11-25T03:37:56,759 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/ff8674afbcb94968b039ec8b7a6323e9, entries=150, sequenceid=12, filesize=11.7 K 2024-11-25T03:37:56,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/67a1b3058b724c33bc2e579763a2258b as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/67a1b3058b724c33bc2e579763a2258b 2024-11-25T03:37:56,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/67a1b3058b724c33bc2e579763a2258b, entries=150, sequenceid=12, filesize=11.7 K 2024-11-25T03:37:56,784 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for c6d2702d0a1f8c4a0fdcb6498eb058d5 in 1396ms, sequenceid=12, compaction requested=false 2024-11-25T03:37:56,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c6d2702d0a1f8c4a0fdcb6498eb058d5: 2024-11-25T03:37:56,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-25T03:37:56,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:37:56,857 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing c6d2702d0a1f8c4a0fdcb6498eb058d5 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-25T03:37:56,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. as already flushing 2024-11-25T03:37:56,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on c6d2702d0a1f8c4a0fdcb6498eb058d5 2024-11-25T03:37:56,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/67835b1a0c68429bb42a58066522e38e is 50, key is test_row_0/A:col10/1732505875718/Put/seqid=0 2024-11-25T03:37:56,990 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T03:37:57,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742075_1251 (size=12001) 2024-11-25T03:37:57,070 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/67835b1a0c68429bb42a58066522e38e 2024-11-25T03:37:57,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:57,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1732505937081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:57,088 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:57,088 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:57,088 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:57,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/75777a61a1a24c5d8ed5904e1c698d92 is 50, key is test_row_0/B:col10/1732505875718/Put/seqid=0 2024-11-25T03:37:57,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:57,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1732505937198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:57,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:57,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:57,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:57,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:57,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57844 deadline: 1732505937217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:57,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:57,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57870 deadline: 1732505937217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:57,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:57,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:57,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:57,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57856 deadline: 1732505937217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:57,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:57,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:57,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:57,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:57,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:57,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:57,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:57,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:57,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505937237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:57,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:57,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:57,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:57,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:57,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1732505937428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:57,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:57,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:57,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:57,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-25T03:37:57,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742076_1252 (size=12001) 2024-11-25T03:37:57,604 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/75777a61a1a24c5d8ed5904e1c698d92 2024-11-25T03:37:57,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/93850fe96194428a8778fa43245765bd is 50, key is test_row_0/C:col10/1732505875718/Put/seqid=0 2024-11-25T03:37:57,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742077_1253 (size=12001) 2024-11-25T03:37:57,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:57,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1732505937792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:57,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:57,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:57,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:58,192 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/93850fe96194428a8778fa43245765bd 2024-11-25T03:37:58,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/67835b1a0c68429bb42a58066522e38e as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/67835b1a0c68429bb42a58066522e38e 2024-11-25T03:37:58,203 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/67835b1a0c68429bb42a58066522e38e, entries=150, sequenceid=38, filesize=11.7 K 2024-11-25T03:37:58,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/75777a61a1a24c5d8ed5904e1c698d92 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/75777a61a1a24c5d8ed5904e1c698d92 2024-11-25T03:37:58,225 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/75777a61a1a24c5d8ed5904e1c698d92, entries=150, sequenceid=38, filesize=11.7 K 2024-11-25T03:37:58,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/93850fe96194428a8778fa43245765bd as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/93850fe96194428a8778fa43245765bd 2024-11-25T03:37:58,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:58,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57870 deadline: 1732505938249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:58,254 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,34881,1732505806588 for 'test_row_1' in c6d2702d0a1f8c4a0fdcb6498eb058d5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2447 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:37:58,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:58,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:58,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:58,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:58,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57856 deadline: 1732505938257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:58,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:58,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57844 deadline: 1732505938257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:58,259 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,34881,1732505806588 for 'test_row_0' in c6d2702d0a1f8c4a0fdcb6498eb058d5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2433 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:37:58,259 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,34881,1732505806588 for 'test_row_2' in c6d2702d0a1f8c4a0fdcb6498eb058d5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2493 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:37:58,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:58,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:58,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:58,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:58,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:58,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:58,261 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/93850fe96194428a8778fa43245765bd, entries=150, sequenceid=38, filesize=11.7 K 2024-11-25T03:37:58,326 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for c6d2702d0a1f8c4a0fdcb6498eb058d5 in 1469ms, sequenceid=38, compaction requested=false 2024-11-25T03:37:58,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for c6d2702d0a1f8c4a0fdcb6498eb058d5: 2024-11-25T03:37:58,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:37:58,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-25T03:37:58,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-25T03:37:58,330 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-25T03:37:58,331 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9890 sec 2024-11-25T03:37:58,333 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 2.9960 sec 2024-11-25T03:37:58,335 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:37:58,335 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c6d2702d0a1f8c4a0fdcb6498eb058d5 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-25T03:37:58,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on c6d2702d0a1f8c4a0fdcb6498eb058d5 2024-11-25T03:37:58,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/6af3d274bb3d4413920c757af6598b2f is 50, key is test_row_0/A:col10/1732505877079/Put/seqid=0 2024-11-25T03:37:58,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742078_1254 (size=14341) 2024-11-25T03:37:58,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/6af3d274bb3d4413920c757af6598b2f 2024-11-25T03:37:58,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:58,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505938877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:58,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:58,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1732505938878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:58,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:58,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:58,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:58,933 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/673c8a12e77a4a63a637dc4eefd877fa is 50, key is test_row_0/B:col10/1732505877079/Put/seqid=0 2024-11-25T03:37:59,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:59,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505938997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:59,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:59,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1732505939024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:59,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:59,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:59,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:59,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:59,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:59,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:59,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742079_1255 (size=12001) 2024-11-25T03:37:59,239 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/673c8a12e77a4a63a637dc4eefd877fa 2024-11-25T03:37:59,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/506b368e16df4068aa92fd523f2199e3 is 50, key is test_row_0/C:col10/1732505877079/Put/seqid=0 2024-11-25T03:37:59,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:59,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1732505939277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:59,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:59,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:59,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:59,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:59,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505939285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:59,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:59,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:59,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:59,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742080_1256 (size=12001) 2024-11-25T03:37:59,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-25T03:37:59,517 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:37:59,518 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:37:59,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-25T03:37:59,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-25T03:37:59,526 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:37:59,527 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:37:59,527 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:37:59,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:59,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505939599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:59,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-25T03:37:59,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:37:59,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1732505939611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:37:59,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:59,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:59,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:59,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:59,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:37:59,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:37:59,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-25T03:37:59,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:37:59,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. as already flushing 2024-11-25T03:37:59,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:37:59,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:59,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:59,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:37:59,797 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/506b368e16df4068aa92fd523f2199e3 2024-11-25T03:37:59,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/6af3d274bb3d4413920c757af6598b2f as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/6af3d274bb3d4413920c757af6598b2f 2024-11-25T03:37:59,809 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/6af3d274bb3d4413920c757af6598b2f, entries=200, sequenceid=49, filesize=14.0 K 2024-11-25T03:37:59,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/673c8a12e77a4a63a637dc4eefd877fa as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/673c8a12e77a4a63a637dc4eefd877fa 2024-11-25T03:37:59,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/673c8a12e77a4a63a637dc4eefd877fa, entries=150, sequenceid=49, filesize=11.7 K 2024-11-25T03:37:59,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/506b368e16df4068aa92fd523f2199e3 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/506b368e16df4068aa92fd523f2199e3 2024-11-25T03:37:59,885 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/506b368e16df4068aa92fd523f2199e3, entries=150, sequenceid=49, filesize=11.7 K 2024-11-25T03:37:59,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-25T03:37:59,888 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for c6d2702d0a1f8c4a0fdcb6498eb058d5 in 1553ms, sequenceid=49, compaction requested=true 2024-11-25T03:37:59,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c6d2702d0a1f8c4a0fdcb6498eb058d5: 2024-11-25T03:37:59,889 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:37:59,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6d2702d0a1f8c4a0fdcb6498eb058d5:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:37:59,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:59,889 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:37:59,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6d2702d0a1f8c4a0fdcb6498eb058d5:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:37:59,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:37:59,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6d2702d0a1f8c4a0fdcb6498eb058d5:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:37:59,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:37:59,890 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:37:59,890 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): c6d2702d0a1f8c4a0fdcb6498eb058d5/B is initiating minor compaction (all files) 2024-11-25T03:37:59,890 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6d2702d0a1f8c4a0fdcb6498eb058d5/B in TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:37:59,891 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/ff8674afbcb94968b039ec8b7a6323e9, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/75777a61a1a24c5d8ed5904e1c698d92, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/673c8a12e77a4a63a637dc4eefd877fa] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp, totalSize=35.2 K 2024-11-25T03:37:59,891 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:37:59,891 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): c6d2702d0a1f8c4a0fdcb6498eb058d5/A is initiating minor compaction (all files) 2024-11-25T03:37:59,891 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6d2702d0a1f8c4a0fdcb6498eb058d5/A in TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:37:59,891 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/6b4bcc156fcf4c99912a3bbc86f9a131, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/67835b1a0c68429bb42a58066522e38e, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/6af3d274bb3d4413920c757af6598b2f] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp, totalSize=37.4 K 2024-11-25T03:37:59,892 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting ff8674afbcb94968b039ec8b7a6323e9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732505875323 2024-11-25T03:37:59,892 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b4bcc156fcf4c99912a3bbc86f9a131, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732505875323 2024-11-25T03:37:59,892 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 75777a61a1a24c5d8ed5904e1c698d92, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732505875718 2024-11-25T03:37:59,893 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67835b1a0c68429bb42a58066522e38e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732505875718 2024-11-25T03:37:59,893 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 673c8a12e77a4a63a637dc4eefd877fa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732505877064 2024-11-25T03:37:59,893 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6af3d274bb3d4413920c757af6598b2f, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732505876968 2024-11-25T03:37:59,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-25T03:37:59,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:37:59,910 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing c6d2702d0a1f8c4a0fdcb6498eb058d5 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-25T03:37:59,942 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6d2702d0a1f8c4a0fdcb6498eb058d5#B#compaction#212 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:37:59,943 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/fde8222fef134fffbe4430bcd68f194d is 50, key is test_row_0/B:col10/1732505877079/Put/seqid=0 2024-11-25T03:37:59,958 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6d2702d0a1f8c4a0fdcb6498eb058d5#A#compaction#213 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:37:59,958 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/c378854276e842d9ab252ef759ca52cc is 50, key is test_row_0/A:col10/1732505877079/Put/seqid=0 2024-11-25T03:37:59,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/21e8530fc0a64bf189e7778a2f8fcdc4 is 50, key is test_row_0/A:col10/1732505878869/Put/seqid=0 2024-11-25T03:38:00,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-25T03:38:00,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. as already flushing 2024-11-25T03:38:00,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on c6d2702d0a1f8c4a0fdcb6498eb058d5 2024-11-25T03:38:00,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742081_1257 (size=12104) 2024-11-25T03:38:00,312 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/fde8222fef134fffbe4430bcd68f194d as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/fde8222fef134fffbe4430bcd68f194d 2024-11-25T03:38:00,323 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c6d2702d0a1f8c4a0fdcb6498eb058d5/B of c6d2702d0a1f8c4a0fdcb6498eb058d5 into fde8222fef134fffbe4430bcd68f194d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:38:00,323 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6d2702d0a1f8c4a0fdcb6498eb058d5: 2024-11-25T03:38:00,323 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., storeName=c6d2702d0a1f8c4a0fdcb6498eb058d5/B, priority=13, startTime=1732505879889; duration=0sec 2024-11-25T03:38:00,323 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:38:00,323 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6d2702d0a1f8c4a0fdcb6498eb058d5:B 2024-11-25T03:38:00,323 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:38:00,325 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:38:00,325 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): c6d2702d0a1f8c4a0fdcb6498eb058d5/C is initiating minor compaction (all files) 2024-11-25T03:38:00,325 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6d2702d0a1f8c4a0fdcb6498eb058d5/C in TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:00,326 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/67a1b3058b724c33bc2e579763a2258b, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/93850fe96194428a8778fa43245765bd, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/506b368e16df4068aa92fd523f2199e3] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp, totalSize=35.2 K 2024-11-25T03:38:00,326 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 67a1b3058b724c33bc2e579763a2258b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732505875323 2024-11-25T03:38:00,326 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 93850fe96194428a8778fa43245765bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732505875718 2024-11-25T03:38:00,327 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 506b368e16df4068aa92fd523f2199e3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732505877064 2024-11-25T03:38:00,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742082_1258 (size=12104) 2024-11-25T03:38:00,378 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/c378854276e842d9ab252ef759ca52cc as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/c378854276e842d9ab252ef759ca52cc 2024-11-25T03:38:00,383 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c6d2702d0a1f8c4a0fdcb6498eb058d5/A of c6d2702d0a1f8c4a0fdcb6498eb058d5 into c378854276e842d9ab252ef759ca52cc(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:38:00,383 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6d2702d0a1f8c4a0fdcb6498eb058d5: 2024-11-25T03:38:00,383 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., storeName=c6d2702d0a1f8c4a0fdcb6498eb058d5/A, priority=13, startTime=1732505879888; duration=0sec 2024-11-25T03:38:00,383 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:38:00,383 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6d2702d0a1f8c4a0fdcb6498eb058d5:A 2024-11-25T03:38:00,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:00,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505940325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:00,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:00,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:00,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:00,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:00,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57870 deadline: 1732505940380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:00,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:00,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57856 deadline: 1732505940387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:00,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:00,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:00,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:00,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:00,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57844 deadline: 1732505940389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:00,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:00,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1732505940401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:00,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:00,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:00,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:00,443 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6d2702d0a1f8c4a0fdcb6498eb058d5#C#compaction#215 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:38:00,444 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/77468b27b66848bfa4d48b18994813c2 is 50, key is test_row_0/C:col10/1732505877079/Put/seqid=0 2024-11-25T03:38:00,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742083_1259 (size=12001) 2024-11-25T03:38:00,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742084_1260 (size=12104) 2024-11-25T03:38:00,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:00,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:00,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:00,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:00,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:00,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:00,507 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/77468b27b66848bfa4d48b18994813c2 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/77468b27b66848bfa4d48b18994813c2 2024-11-25T03:38:00,518 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c6d2702d0a1f8c4a0fdcb6498eb058d5/C of c6d2702d0a1f8c4a0fdcb6498eb058d5 into 77468b27b66848bfa4d48b18994813c2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:38:00,518 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6d2702d0a1f8c4a0fdcb6498eb058d5: 2024-11-25T03:38:00,518 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., storeName=c6d2702d0a1f8c4a0fdcb6498eb058d5/C, priority=13, startTime=1732505879889; duration=0sec 2024-11-25T03:38:00,518 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:38:00,518 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6d2702d0a1f8c4a0fdcb6498eb058d5:C 2024-11-25T03:38:00,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:00,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505940520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:00,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:00,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:00,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:00,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:00,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1732505940538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:00,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:00,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57856 deadline: 1732505940538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:00,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:00,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:00,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:00,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:00,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:00,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:00,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:00,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57870 deadline: 1732505940569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:00,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:00,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:00,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:00,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:00,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57844 deadline: 1732505940572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:00,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:00,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:00,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:00,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-25T03:38:00,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:00,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505940781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:00,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:00,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:00,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:00,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:00,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57856 deadline: 1732505940827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:00,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:00,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:00,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:00,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:00,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1732505940828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:00,845 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/21e8530fc0a64bf189e7778a2f8fcdc4 2024-11-25T03:38:00,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:00,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:00,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:00,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:00,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57844 deadline: 1732505940828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:00,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:00,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:00,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:00,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:00,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57870 deadline: 1732505940828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:00,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:00,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:00,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:00,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/8d42727018454c02b1263e3f715e6fc7 is 50, key is test_row_0/B:col10/1732505878869/Put/seqid=0 2024-11-25T03:38:01,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742085_1261 (size=12001) 2024-11-25T03:38:01,019 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/8d42727018454c02b1263e3f715e6fc7 2024-11-25T03:38:01,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/934d0c6c6f274a73adab01b5f52924cd is 50, key is test_row_0/C:col10/1732505878869/Put/seqid=0 2024-11-25T03:38:01,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742086_1262 (size=12001) 2024-11-25T03:38:01,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:01,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505941105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:01,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:01,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57844 deadline: 1732505941157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:01,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:01,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:01,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:01,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:01,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57870 deadline: 1732505941157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:01,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:01,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57856 deadline: 1732505941185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:01,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:01,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:01,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:01,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:01,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1732505941185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:01,248 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:01,248 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:01,248 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:01,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:01,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:01,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:01,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:01,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:01,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:01,484 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/934d0c6c6f274a73adab01b5f52924cd 2024-11-25T03:38:01,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/21e8530fc0a64bf189e7778a2f8fcdc4 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/21e8530fc0a64bf189e7778a2f8fcdc4 2024-11-25T03:38:01,496 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/21e8530fc0a64bf189e7778a2f8fcdc4, entries=150, sequenceid=75, filesize=11.7 K 2024-11-25T03:38:01,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/8d42727018454c02b1263e3f715e6fc7 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/8d42727018454c02b1263e3f715e6fc7 2024-11-25T03:38:01,502 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/8d42727018454c02b1263e3f715e6fc7, entries=150, sequenceid=75, filesize=11.7 K 2024-11-25T03:38:01,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/934d0c6c6f274a73adab01b5f52924cd as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/934d0c6c6f274a73adab01b5f52924cd 2024-11-25T03:38:01,507 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/934d0c6c6f274a73adab01b5f52924cd, entries=150, sequenceid=75, filesize=11.7 K 2024-11-25T03:38:01,517 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for c6d2702d0a1f8c4a0fdcb6498eb058d5 in 1607ms, sequenceid=75, compaction requested=false 2024-11-25T03:38:01,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for c6d2702d0a1f8c4a0fdcb6498eb058d5: 2024-11-25T03:38:01,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:01,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-25T03:38:01,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-25T03:38:01,521 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-25T03:38:01,521 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9920 sec 2024-11-25T03:38:01,525 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 2.0040 sec 2024-11-25T03:38:01,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-25T03:38:01,719 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:38:01,721 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:38:01,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-25T03:38:01,723 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:38:01,724 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:38:01,725 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:38:01,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-25T03:38:01,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-25T03:38:01,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-25T03:38:01,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:01,927 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing c6d2702d0a1f8c4a0fdcb6498eb058d5 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-25T03:38:01,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. as already flushing 2024-11-25T03:38:01,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on c6d2702d0a1f8c4a0fdcb6498eb058d5 2024-11-25T03:38:01,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/96f8c6ab33d84e5f84d1d0f0b6eaff88 is 50, key is test_row_0/A:col10/1732505880387/Put/seqid=0 2024-11-25T03:38:01,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742087_1263 (size=12001) 2024-11-25T03:38:01,970 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/96f8c6ab33d84e5f84d1d0f0b6eaff88 2024-11-25T03:38:02,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-25T03:38:02,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/504a74083c7048e5a577564e770d2306 is 50, key is test_row_0/B:col10/1732505880387/Put/seqid=0 2024-11-25T03:38:02,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742088_1264 (size=12001) 2024-11-25T03:38:02,170 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/504a74083c7048e5a577564e770d2306 2024-11-25T03:38:02,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:02,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57856 deadline: 1732505942168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:02,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:02,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:02,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:02,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:02,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505942163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:02,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:02,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:02,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:02,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:02,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1732505942173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:02,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:02,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:02,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:02,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/860f0710163247089acb8ad7122ecb8e is 50, key is test_row_0/C:col10/1732505880387/Put/seqid=0 2024-11-25T03:38:02,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:02,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57870 deadline: 1732505942177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:02,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:02,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:02,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:02,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:02,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57844 deadline: 1732505942171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:02,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:02,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:02,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:02,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:02,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57856 deadline: 1732505942297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:02,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-25T03:38:02,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:02,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:02,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:02,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:02,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505942309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:02,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:02,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1732505942309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:02,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:02,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:02,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:02,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:02,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:02,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:02,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:02,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57870 deadline: 1732505942337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:02,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:02,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:02,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:02,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:02,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57844 deadline: 1732505942421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:02,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:02,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:02,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:02,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742089_1265 (size=12001) 2024-11-25T03:38:02,571 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/860f0710163247089acb8ad7122ecb8e 2024-11-25T03:38:02,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/96f8c6ab33d84e5f84d1d0f0b6eaff88 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/96f8c6ab33d84e5f84d1d0f0b6eaff88 2024-11-25T03:38:02,590 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/96f8c6ab33d84e5f84d1d0f0b6eaff88, entries=150, sequenceid=88, filesize=11.7 K 2024-11-25T03:38:02,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/504a74083c7048e5a577564e770d2306 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/504a74083c7048e5a577564e770d2306 2024-11-25T03:38:02,596 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/504a74083c7048e5a577564e770d2306, entries=150, sequenceid=88, filesize=11.7 K 2024-11-25T03:38:02,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/860f0710163247089acb8ad7122ecb8e as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/860f0710163247089acb8ad7122ecb8e 2024-11-25T03:38:02,601 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/860f0710163247089acb8ad7122ecb8e, entries=150, sequenceid=88, filesize=11.7 K 2024-11-25T03:38:02,602 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=161.02 KB/164880 for c6d2702d0a1f8c4a0fdcb6498eb058d5 in 676ms, sequenceid=88, compaction requested=true 2024-11-25T03:38:02,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for c6d2702d0a1f8c4a0fdcb6498eb058d5: 2024-11-25T03:38:02,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:02,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-25T03:38:02,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-25T03:38:02,606 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-25T03:38:02,606 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 880 msec 2024-11-25T03:38:02,609 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 886 msec 2024-11-25T03:38:02,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on c6d2702d0a1f8c4a0fdcb6498eb058d5 2024-11-25T03:38:02,653 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:38:02,653 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c6d2702d0a1f8c4a0fdcb6498eb058d5 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-25T03:38:02,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/966d63dd5e4445a1a1389d6f89b05d45 is 50, key is test_row_0/A:col10/1732505882172/Put/seqid=0 2024-11-25T03:38:02,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:02,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1732505942718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:02,725 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:02,725 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:02,725 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:02,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:02,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57870 deadline: 1732505942719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:02,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:02,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:02,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:02,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:02,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505942719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:02,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:02,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:02,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:02,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:02,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57856 deadline: 1732505942724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:02,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:02,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57844 deadline: 1732505942725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:02,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:02,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:02,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:02,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:02,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:02,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:02,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742090_1266 (size=14341) 2024-11-25T03:38:02,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/966d63dd5e4445a1a1389d6f89b05d45 2024-11-25T03:38:02,813 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/ed347874d15d44c9bc5a873a8c32818b is 50, key is test_row_0/B:col10/1732505882172/Put/seqid=0 2024-11-25T03:38:02,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742091_1267 (size=12001) 2024-11-25T03:38:02,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:02,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1732505942827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:02,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:02,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:02,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57870 deadline: 1732505942838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:02,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:02,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:02,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:02,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:02,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:02,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-25T03:38:02,857 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:38:02,858 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:38:02,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-25T03:38:02,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:02,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505942856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:02,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:02,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:02,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:02,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-25T03:38:02,861 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:38:02,862 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:38:02,862 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:38:02,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:02,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57844 deadline: 1732505942900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:02,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:02,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57856 deadline: 1732505942904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:02,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:02,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:02,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:02,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:02,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:02,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:02,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-25T03:38:03,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-25T03:38:03,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:03,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. as already flushing 2024-11-25T03:38:03,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:03,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:03,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:03,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:03,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:03,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1732505943048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:03,097 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:03,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:03,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:03,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:03,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505943067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:03,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:03,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57870 deadline: 1732505943084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:03,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:03,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:03,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:03,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-25T03:38:03,229 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/ed347874d15d44c9bc5a873a8c32818b 2024-11-25T03:38:03,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:03,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:03,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:03,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-25T03:38:03,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:03,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. as already flushing 2024-11-25T03:38:03,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:03,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:03,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:03,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:03,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57844 deadline: 1732505943234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:03,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:03,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:03,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:03,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:03,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:03,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57856 deadline: 1732505943235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:03,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:03,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:03,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:03,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/70d63b66c8f041eb839791c92954a3fb is 50, key is test_row_0/C:col10/1732505882172/Put/seqid=0 2024-11-25T03:38:03,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742092_1268 (size=12001) 2024-11-25T03:38:03,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-25T03:38:03,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:03,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. as already flushing 2024-11-25T03:38:03,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:03,398 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:03,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:03,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:03,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:03,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1732505943407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:03,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:03,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:03,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:03,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:03,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505943418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:03,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:03,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:03,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:03,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-25T03:38:03,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:03,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57870 deadline: 1732505943552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:03,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:03,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57844 deadline: 1732505943552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:03,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:03,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:03,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:03,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:03,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:03,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:03,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-25T03:38:03,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:03,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. as already flushing 2024-11-25T03:38:03,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:03,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:03,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:03,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:03,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:03,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57856 deadline: 1732505943644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:03,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:03,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:03,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:03,731 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/70d63b66c8f041eb839791c92954a3fb 2024-11-25T03:38:03,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/966d63dd5e4445a1a1389d6f89b05d45 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/966d63dd5e4445a1a1389d6f89b05d45 2024-11-25T03:38:03,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/966d63dd5e4445a1a1389d6f89b05d45, entries=200, sequenceid=116, filesize=14.0 K 2024-11-25T03:38:03,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/ed347874d15d44c9bc5a873a8c32818b as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/ed347874d15d44c9bc5a873a8c32818b 2024-11-25T03:38:03,747 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/ed347874d15d44c9bc5a873a8c32818b, entries=150, sequenceid=116, filesize=11.7 K 2024-11-25T03:38:03,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/70d63b66c8f041eb839791c92954a3fb as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/70d63b66c8f041eb839791c92954a3fb 2024-11-25T03:38:03,752 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/70d63b66c8f041eb839791c92954a3fb, entries=150, sequenceid=116, filesize=11.7 K 2024-11-25T03:38:03,756 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for c6d2702d0a1f8c4a0fdcb6498eb058d5 in 1103ms, sequenceid=116, compaction requested=true 2024-11-25T03:38:03,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c6d2702d0a1f8c4a0fdcb6498eb058d5: 2024-11-25T03:38:03,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6d2702d0a1f8c4a0fdcb6498eb058d5:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:38:03,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:38:03,757 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:38:03,757 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:38:03,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6d2702d0a1f8c4a0fdcb6498eb058d5:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:38:03,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:38:03,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6d2702d0a1f8c4a0fdcb6498eb058d5:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:38:03,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:38:03,759 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:38:03,759 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): c6d2702d0a1f8c4a0fdcb6498eb058d5/B is initiating minor compaction (all files) 2024-11-25T03:38:03,759 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6d2702d0a1f8c4a0fdcb6498eb058d5/B in TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:03,759 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/fde8222fef134fffbe4430bcd68f194d, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/8d42727018454c02b1263e3f715e6fc7, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/504a74083c7048e5a577564e770d2306, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/ed347874d15d44c9bc5a873a8c32818b] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp, totalSize=47.0 K 2024-11-25T03:38:03,759 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50447 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:38:03,759 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): c6d2702d0a1f8c4a0fdcb6498eb058d5/A is initiating minor compaction (all files) 2024-11-25T03:38:03,759 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6d2702d0a1f8c4a0fdcb6498eb058d5/A in TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:03,759 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/c378854276e842d9ab252ef759ca52cc, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/21e8530fc0a64bf189e7778a2f8fcdc4, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/96f8c6ab33d84e5f84d1d0f0b6eaff88, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/966d63dd5e4445a1a1389d6f89b05d45] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp, totalSize=49.3 K 2024-11-25T03:38:03,760 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting fde8222fef134fffbe4430bcd68f194d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732505877064 2024-11-25T03:38:03,760 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting c378854276e842d9ab252ef759ca52cc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732505877064 2024-11-25T03:38:03,761 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21e8530fc0a64bf189e7778a2f8fcdc4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732505878839 2024-11-25T03:38:03,761 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d42727018454c02b1263e3f715e6fc7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732505878839 2024-11-25T03:38:03,761 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 504a74083c7048e5a577564e770d2306, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732505880379 2024-11-25T03:38:03,761 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96f8c6ab33d84e5f84d1d0f0b6eaff88, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732505880379 2024-11-25T03:38:03,762 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 966d63dd5e4445a1a1389d6f89b05d45, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732505882096 2024-11-25T03:38:03,762 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting ed347874d15d44c9bc5a873a8c32818b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732505882096 2024-11-25T03:38:03,792 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6d2702d0a1f8c4a0fdcb6498eb058d5#A#compaction#224 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:38:03,793 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/fd41e03de24546fa9344b3a8ee505a51 is 50, key is test_row_0/A:col10/1732505882172/Put/seqid=0 2024-11-25T03:38:03,839 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6d2702d0a1f8c4a0fdcb6498eb058d5#B#compaction#225 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:38:03,840 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/3340461bd28d4460980a317a0844330a is 50, key is test_row_0/B:col10/1732505882172/Put/seqid=0 2024-11-25T03:38:03,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-25T03:38:03,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:03,877 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing c6d2702d0a1f8c4a0fdcb6498eb058d5 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-25T03:38:03,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742093_1269 (size=12241) 2024-11-25T03:38:03,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/9b4c03842fab420ca462efb8c1102ac8 is 50, key is test_row_0/A:col10/1732505882722/Put/seqid=0 2024-11-25T03:38:03,909 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/fd41e03de24546fa9344b3a8ee505a51 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/fd41e03de24546fa9344b3a8ee505a51 2024-11-25T03:38:03,915 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c6d2702d0a1f8c4a0fdcb6498eb058d5/A of c6d2702d0a1f8c4a0fdcb6498eb058d5 into fd41e03de24546fa9344b3a8ee505a51(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:38:03,915 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6d2702d0a1f8c4a0fdcb6498eb058d5: 2024-11-25T03:38:03,915 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., storeName=c6d2702d0a1f8c4a0fdcb6498eb058d5/A, priority=12, startTime=1732505883756; duration=0sec 2024-11-25T03:38:03,915 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:38:03,915 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6d2702d0a1f8c4a0fdcb6498eb058d5:A 2024-11-25T03:38:03,915 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:38:03,916 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:38:03,916 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): c6d2702d0a1f8c4a0fdcb6498eb058d5/C is initiating minor compaction (all files) 2024-11-25T03:38:03,917 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6d2702d0a1f8c4a0fdcb6498eb058d5/C in TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:03,917 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/77468b27b66848bfa4d48b18994813c2, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/934d0c6c6f274a73adab01b5f52924cd, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/860f0710163247089acb8ad7122ecb8e, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/70d63b66c8f041eb839791c92954a3fb] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp, totalSize=47.0 K 2024-11-25T03:38:03,917 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77468b27b66848bfa4d48b18994813c2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732505877064 2024-11-25T03:38:03,917 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 934d0c6c6f274a73adab01b5f52924cd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732505878839 2024-11-25T03:38:03,918 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 860f0710163247089acb8ad7122ecb8e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732505880379 2024-11-25T03:38:03,918 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70d63b66c8f041eb839791c92954a3fb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732505882096 2024-11-25T03:38:03,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-25T03:38:04,118 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6d2702d0a1f8c4a0fdcb6498eb058d5#C#compaction#227 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:38:04,119 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/d584c43b2c364b59a521d3caeb0c8b61 is 50, key is test_row_0/C:col10/1732505882172/Put/seqid=0 2024-11-25T03:38:04,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. as already flushing 2024-11-25T03:38:04,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on c6d2702d0a1f8c4a0fdcb6498eb058d5 2024-11-25T03:38:04,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742094_1270 (size=12241) 2024-11-25T03:38:04,283 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/3340461bd28d4460980a317a0844330a as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/3340461bd28d4460980a317a0844330a 2024-11-25T03:38:04,288 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c6d2702d0a1f8c4a0fdcb6498eb058d5/B of c6d2702d0a1f8c4a0fdcb6498eb058d5 into 3340461bd28d4460980a317a0844330a(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:38:04,288 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6d2702d0a1f8c4a0fdcb6498eb058d5: 2024-11-25T03:38:04,288 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., storeName=c6d2702d0a1f8c4a0fdcb6498eb058d5/B, priority=12, startTime=1732505883757; duration=0sec 2024-11-25T03:38:04,289 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:38:04,289 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6d2702d0a1f8c4a0fdcb6498eb058d5:B 2024-11-25T03:38:04,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742095_1271 (size=9657) 2024-11-25T03:38:04,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742096_1272 (size=12241) 2024-11-25T03:38:04,449 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/d584c43b2c364b59a521d3caeb0c8b61 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/d584c43b2c364b59a521d3caeb0c8b61 2024-11-25T03:38:04,455 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c6d2702d0a1f8c4a0fdcb6498eb058d5/C of c6d2702d0a1f8c4a0fdcb6498eb058d5 into d584c43b2c364b59a521d3caeb0c8b61(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:38:04,455 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6d2702d0a1f8c4a0fdcb6498eb058d5: 2024-11-25T03:38:04,455 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., storeName=c6d2702d0a1f8c4a0fdcb6498eb058d5/C, priority=12, startTime=1732505883757; duration=0sec 2024-11-25T03:38:04,455 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:38:04,455 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6d2702d0a1f8c4a0fdcb6498eb058d5:C 2024-11-25T03:38:04,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:04,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505944470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:04,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:04,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:04,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:04,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:04,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1732505944472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:04,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:04,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:04,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57856 deadline: 1732505944480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:04,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:04,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:04,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:04,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57844 deadline: 1732505944482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:04,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:04,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:04,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:04,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:04,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57870 deadline: 1732505944483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:04,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:04,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:04,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:04,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:04,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:04,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:04,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:04,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505944601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:04,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:04,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:04,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:04,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:04,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1732505944603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:04,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:04,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:04,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:04,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:04,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57870 deadline: 1732505944607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:04,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:04,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:04,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:04,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:04,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57844 deadline: 1732505944607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:04,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:04,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:04,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:04,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:04,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57856 deadline: 1732505944607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:04,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:04,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:04,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:04,755 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/9b4c03842fab420ca462efb8c1102ac8 2024-11-25T03:38:04,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/56b512508579434f9764abb04eb7834c is 50, key is test_row_0/B:col10/1732505882722/Put/seqid=0 2024-11-25T03:38:04,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:04,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505944817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:04,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:04,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:04,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:04,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:04,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1732505944868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:04,877 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:04,877 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:04,877 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:04,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742097_1273 (size=9657) 2024-11-25T03:38:04,972 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/56b512508579434f9764abb04eb7834c 2024-11-25T03:38:04,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:04,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57870 deadline: 1732505944958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:04,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:04,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57844 deadline: 1732505944958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:04,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:04,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57856 deadline: 1732505944958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:04,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:04,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:04,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:05,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-25T03:38:05,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:05,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:05,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:05,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:05,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:05,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:05,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/b2b608f83bd84a4d9015addd9e6c5981 is 50, key is test_row_0/C:col10/1732505882722/Put/seqid=0 2024-11-25T03:38:05,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742098_1274 (size=9657) 2024-11-25T03:38:05,151 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/b2b608f83bd84a4d9015addd9e6c5981 2024-11-25T03:38:05,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/9b4c03842fab420ca462efb8c1102ac8 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/9b4c03842fab420ca462efb8c1102ac8 2024-11-25T03:38:05,176 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/9b4c03842fab420ca462efb8c1102ac8, entries=100, sequenceid=124, filesize=9.4 K 2024-11-25T03:38:05,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/56b512508579434f9764abb04eb7834c as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/56b512508579434f9764abb04eb7834c 2024-11-25T03:38:05,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:05,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1732505945187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:05,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:05,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505945196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:05,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:05,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:05,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:05,234 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/56b512508579434f9764abb04eb7834c, entries=100, sequenceid=124, filesize=9.4 K 2024-11-25T03:38:05,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/b2b608f83bd84a4d9015addd9e6c5981 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/b2b608f83bd84a4d9015addd9e6c5981 2024-11-25T03:38:05,239 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/b2b608f83bd84a4d9015addd9e6c5981, entries=100, sequenceid=124, filesize=9.4 K 2024-11-25T03:38:05,241 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for c6d2702d0a1f8c4a0fdcb6498eb058d5 in 1363ms, sequenceid=124, compaction requested=false 2024-11-25T03:38:05,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for c6d2702d0a1f8c4a0fdcb6498eb058d5: 2024-11-25T03:38:05,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:05,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-25T03:38:05,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-25T03:38:05,245 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-25T03:38:05,245 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3810 sec 2024-11-25T03:38:05,247 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 2.3870 sec 2024-11-25T03:38:05,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:05,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:05,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:05,401 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:38:05,401 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c6d2702d0a1f8c4a0fdcb6498eb058d5 3/3 column families, dataSize=181.14 KB heapSize=475.36 KB 2024-11-25T03:38:05,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on c6d2702d0a1f8c4a0fdcb6498eb058d5 2024-11-25T03:38:05,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/4952b9ff5c024c38a4088aa176ab1631 is 50, key is test_row_0/A:col10/1732505884470/Put/seqid=0 2024-11-25T03:38:05,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:05,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57856 deadline: 1732505945473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:05,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:05,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:05,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:05,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:05,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57870 deadline: 1732505945463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:05,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:05,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:05,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:05,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:05,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57844 deadline: 1732505945517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:05,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:05,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:05,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:05,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742099_1275 (size=12151) 2024-11-25T03:38:05,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:05,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:05,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57856 deadline: 1732505945636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:05,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57870 deadline: 1732505945632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:05,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:05,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:05,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:05,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:05,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:05,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:05,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:05,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505945737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:05,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:05,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:05,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:05,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:05,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57844 deadline: 1732505945658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:05,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:05,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:05,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:05,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:05,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1732505945800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:05,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:05,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:05,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:05,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:05,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57870 deadline: 1732505945957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:05,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:05,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:05,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:05,995 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/4952b9ff5c024c38a4088aa176ab1631 2024-11-25T03:38:06,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:06,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57856 deadline: 1732505946008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:06,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:06,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:06,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:06,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57844 deadline: 1732505946027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:06,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/773d5fb95697427ab1d0dd17a204a92d is 50, key is test_row_0/B:col10/1732505884470/Put/seqid=0 2024-11-25T03:38:06,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:06,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:06,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:06,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742100_1276 (size=12151) 2024-11-25T03:38:06,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57870 deadline: 1732505946277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:06,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:06,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:06,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:06,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:06,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57856 deadline: 1732505946318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:06,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:06,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:06,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:06,365 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/773d5fb95697427ab1d0dd17a204a92d 2024-11-25T03:38:06,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:06,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57844 deadline: 1732505946398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:06,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:06,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:06,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:06,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/46a9df7a45c140bd8dc5cceb90d1543c is 50, key is test_row_0/C:col10/1732505884470/Put/seqid=0 2024-11-25T03:38:06,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742101_1277 (size=12151) 2024-11-25T03:38:06,686 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/46a9df7a45c140bd8dc5cceb90d1543c 2024-11-25T03:38:06,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/4952b9ff5c024c38a4088aa176ab1631 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/4952b9ff5c024c38a4088aa176ab1631 2024-11-25T03:38:06,705 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/4952b9ff5c024c38a4088aa176ab1631, entries=150, sequenceid=157, filesize=11.9 K 2024-11-25T03:38:06,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/773d5fb95697427ab1d0dd17a204a92d as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/773d5fb95697427ab1d0dd17a204a92d 2024-11-25T03:38:06,711 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/773d5fb95697427ab1d0dd17a204a92d, entries=150, sequenceid=157, filesize=11.9 K 2024-11-25T03:38:06,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/46a9df7a45c140bd8dc5cceb90d1543c as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/46a9df7a45c140bd8dc5cceb90d1543c 2024-11-25T03:38:06,716 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/46a9df7a45c140bd8dc5cceb90d1543c, entries=150, sequenceid=157, filesize=11.9 K 2024-11-25T03:38:06,717 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=26.84 KB/27480 for c6d2702d0a1f8c4a0fdcb6498eb058d5 in 1316ms, sequenceid=157, compaction requested=true 2024-11-25T03:38:06,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c6d2702d0a1f8c4a0fdcb6498eb058d5: 2024-11-25T03:38:06,717 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:38:06,719 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:38:06,719 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): c6d2702d0a1f8c4a0fdcb6498eb058d5/A is initiating minor compaction (all files) 2024-11-25T03:38:06,719 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6d2702d0a1f8c4a0fdcb6498eb058d5/A in TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:06,719 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/fd41e03de24546fa9344b3a8ee505a51, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/9b4c03842fab420ca462efb8c1102ac8, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/4952b9ff5c024c38a4088aa176ab1631] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp, totalSize=33.3 K 2024-11-25T03:38:06,720 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd41e03de24546fa9344b3a8ee505a51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732505882096 2024-11-25T03:38:06,720 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b4c03842fab420ca462efb8c1102ac8, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1732505882722 2024-11-25T03:38:06,720 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4952b9ff5c024c38a4088aa176ab1631, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732505884470 2024-11-25T03:38:06,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6d2702d0a1f8c4a0fdcb6498eb058d5:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:38:06,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:38:06,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6d2702d0a1f8c4a0fdcb6498eb058d5:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:38:06,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:38:06,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6d2702d0a1f8c4a0fdcb6498eb058d5:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:38:06,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:38:06,721 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:38:06,722 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:38:06,722 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): c6d2702d0a1f8c4a0fdcb6498eb058d5/B is initiating minor compaction (all files) 2024-11-25T03:38:06,722 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6d2702d0a1f8c4a0fdcb6498eb058d5/B in TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:06,723 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/3340461bd28d4460980a317a0844330a, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/56b512508579434f9764abb04eb7834c, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/773d5fb95697427ab1d0dd17a204a92d] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp, totalSize=33.3 K 2024-11-25T03:38:06,723 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 3340461bd28d4460980a317a0844330a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732505882096 2024-11-25T03:38:06,723 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 56b512508579434f9764abb04eb7834c, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1732505882722 2024-11-25T03:38:06,724 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 773d5fb95697427ab1d0dd17a204a92d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732505884470 2024-11-25T03:38:06,729 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6d2702d0a1f8c4a0fdcb6498eb058d5#A#compaction#233 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:38:06,733 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/e7f25cc9a0cb4586b43abd2479a1dc89 is 50, key is test_row_0/A:col10/1732505884470/Put/seqid=0 2024-11-25T03:38:06,733 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6d2702d0a1f8c4a0fdcb6498eb058d5#B#compaction#234 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:38:06,734 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/5ac60418511549b2b7a5cccbbd7a6b2f is 50, key is test_row_0/B:col10/1732505884470/Put/seqid=0 2024-11-25T03:38:06,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742102_1278 (size=12493) 2024-11-25T03:38:06,786 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/e7f25cc9a0cb4586b43abd2479a1dc89 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/e7f25cc9a0cb4586b43abd2479a1dc89 2024-11-25T03:38:06,792 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c6d2702d0a1f8c4a0fdcb6498eb058d5/A of c6d2702d0a1f8c4a0fdcb6498eb058d5 into e7f25cc9a0cb4586b43abd2479a1dc89(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:38:06,793 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6d2702d0a1f8c4a0fdcb6498eb058d5: 2024-11-25T03:38:06,793 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., storeName=c6d2702d0a1f8c4a0fdcb6498eb058d5/A, priority=13, startTime=1732505886717; duration=0sec 2024-11-25T03:38:06,793 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:38:06,793 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6d2702d0a1f8c4a0fdcb6498eb058d5:A 2024-11-25T03:38:06,793 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:38:06,794 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:38:06,794 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): c6d2702d0a1f8c4a0fdcb6498eb058d5/C is initiating minor compaction (all files) 2024-11-25T03:38:06,794 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6d2702d0a1f8c4a0fdcb6498eb058d5/C in TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:06,794 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/d584c43b2c364b59a521d3caeb0c8b61, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/b2b608f83bd84a4d9015addd9e6c5981, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/46a9df7a45c140bd8dc5cceb90d1543c] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp, totalSize=33.3 K 2024-11-25T03:38:06,795 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting d584c43b2c364b59a521d3caeb0c8b61, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732505882096 2024-11-25T03:38:06,795 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2b608f83bd84a4d9015addd9e6c5981, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1732505882722 2024-11-25T03:38:06,796 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46a9df7a45c140bd8dc5cceb90d1543c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732505884470 2024-11-25T03:38:06,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on c6d2702d0a1f8c4a0fdcb6498eb058d5 2024-11-25T03:38:06,878 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:38:06,878 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c6d2702d0a1f8c4a0fdcb6498eb058d5 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-25T03:38:06,880 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6d2702d0a1f8c4a0fdcb6498eb058d5#C#compaction#235 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:38:06,880 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/c50ea88784d94720b08b8d3b6bec91a1 is 50, key is test_row_0/C:col10/1732505884470/Put/seqid=0 2024-11-25T03:38:06,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/bc17a0bef87f484fb1eedf95a66e149b is 50, key is test_row_0/A:col10/1732505886847/Put/seqid=0 2024-11-25T03:38:06,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742103_1279 (size=12493) 2024-11-25T03:38:06,945 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/5ac60418511549b2b7a5cccbbd7a6b2f as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/5ac60418511549b2b7a5cccbbd7a6b2f 2024-11-25T03:38:06,952 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c6d2702d0a1f8c4a0fdcb6498eb058d5/B of c6d2702d0a1f8c4a0fdcb6498eb058d5 into 5ac60418511549b2b7a5cccbbd7a6b2f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:38:06,952 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6d2702d0a1f8c4a0fdcb6498eb058d5: 2024-11-25T03:38:06,952 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., storeName=c6d2702d0a1f8c4a0fdcb6498eb058d5/B, priority=13, startTime=1732505886721; duration=0sec 2024-11-25T03:38:06,952 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:38:06,952 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6d2702d0a1f8c4a0fdcb6498eb058d5:B 2024-11-25T03:38:07,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-25T03:38:07,017 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:38:07,018 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:38:07,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-25T03:38:07,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-25T03:38:07,022 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:38:07,023 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:38:07,023 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:38:07,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742104_1280 (size=12151) 2024-11-25T03:38:07,103 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/bc17a0bef87f484fb1eedf95a66e149b 2024-11-25T03:38:07,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/b56b9ccfd2b548779dae3e7b4137b812 is 50, key is test_row_0/B:col10/1732505886847/Put/seqid=0 2024-11-25T03:38:07,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-25T03:38:07,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-25T03:38:07,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:07,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. as already flushing 2024-11-25T03:38:07,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:07,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:07,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:07,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742105_1281 (size=12493) 2024-11-25T03:38:07,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:07,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:07,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57856 deadline: 1732505947160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:07,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:07,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:07,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:07,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:07,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505947163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:07,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:07,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:07,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:07,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:07,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1732505947164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:07,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:07,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:07,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:07,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:07,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57844 deadline: 1732505947199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:07,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:07,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:07,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:07,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:07,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57870 deadline: 1732505947210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:07,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:07,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:07,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:07,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:07,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57856 deadline: 1732505947317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:07,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:07,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:07,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:07,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:07,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1732505947328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:07,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-25T03:38:07,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:07,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:07,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:07,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-25T03:38:07,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:07,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. as already flushing 2024-11-25T03:38:07,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:07,486 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:07,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:07,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:07,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505947328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:07,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742106_1282 (size=12151) 2024-11-25T03:38:07,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:07,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:07,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57844 deadline: 1732505947328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:07,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:07,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:07,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:07,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:07,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:07,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:07,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:07,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57870 deadline: 1732505947338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:07,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:07,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:07,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:07,584 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/c50ea88784d94720b08b8d3b6bec91a1 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/c50ea88784d94720b08b8d3b6bec91a1 2024-11-25T03:38:07,589 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c6d2702d0a1f8c4a0fdcb6498eb058d5/C of c6d2702d0a1f8c4a0fdcb6498eb058d5 into c50ea88784d94720b08b8d3b6bec91a1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:38:07,589 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6d2702d0a1f8c4a0fdcb6498eb058d5: 2024-11-25T03:38:07,589 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., storeName=c6d2702d0a1f8c4a0fdcb6498eb058d5/C, priority=13, startTime=1732505886721; duration=0sec 2024-11-25T03:38:07,589 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:38:07,589 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6d2702d0a1f8c4a0fdcb6498eb058d5:C 2024-11-25T03:38:07,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-25T03:38:07,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:07,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. as already flushing 2024-11-25T03:38:07,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:07,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:07,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:07,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:07,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57856 deadline: 1732505947576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:07,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:07,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:07,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1732505947577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:07,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:07,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:07,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:07,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:07,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:07,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:07,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-25T03:38:07,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:07,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57844 deadline: 1732505947707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:07,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:07,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:07,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:07,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:07,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57870 deadline: 1732505947717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:07,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:07,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:07,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:07,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:07,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505947741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:07,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:07,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:07,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:07,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-25T03:38:07,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:07,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. as already flushing 2024-11-25T03:38:07,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:07,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:07,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:07,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:07,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/b56b9ccfd2b548779dae3e7b4137b812 2024-11-25T03:38:07,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/29e2971d4ae443cc83491d0dcb62f894 is 50, key is test_row_0/C:col10/1732505886847/Put/seqid=0 2024-11-25T03:38:07,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-25T03:38:07,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:07,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. as already flushing 2024-11-25T03:38:07,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:07,986 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:07,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:08,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:08,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:08,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57856 deadline: 1732505947960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:08,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:08,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:08,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:08,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1732505947960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:08,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:08,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:08,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:08,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:08,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:08,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57844 deadline: 1732505948027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:08,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:08,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:08,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:08,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742107_1283 (size=12151) 2024-11-25T03:38:08,092 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/29e2971d4ae443cc83491d0dcb62f894 2024-11-25T03:38:08,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/bc17a0bef87f484fb1eedf95a66e149b as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/bc17a0bef87f484fb1eedf95a66e149b 2024-11-25T03:38:08,102 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/bc17a0bef87f484fb1eedf95a66e149b, entries=150, sequenceid=169, filesize=11.9 K 2024-11-25T03:38:08,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/b56b9ccfd2b548779dae3e7b4137b812 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/b56b9ccfd2b548779dae3e7b4137b812 2024-11-25T03:38:08,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-25T03:38:08,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-25T03:38:08,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:08,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. as already flushing 2024-11-25T03:38:08,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:08,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:08,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:08,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/b56b9ccfd2b548779dae3e7b4137b812, entries=150, sequenceid=169, filesize=11.9 K 2024-11-25T03:38:08,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:08,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/29e2971d4ae443cc83491d0dcb62f894 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/29e2971d4ae443cc83491d0dcb62f894 2024-11-25T03:38:08,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/29e2971d4ae443cc83491d0dcb62f894, entries=150, sequenceid=169, filesize=11.9 K 2024-11-25T03:38:08,263 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for c6d2702d0a1f8c4a0fdcb6498eb058d5 in 1385ms, sequenceid=169, compaction requested=false 2024-11-25T03:38:08,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c6d2702d0a1f8c4a0fdcb6498eb058d5: 2024-11-25T03:38:08,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on c6d2702d0a1f8c4a0fdcb6498eb058d5 2024-11-25T03:38:08,277 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:38:08,277 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c6d2702d0a1f8c4a0fdcb6498eb058d5 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-25T03:38:08,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/13461dee72a24f809b286de242003a4b is 50, key is test_row_0/A:col10/1732505888272/Put/seqid=0 2024-11-25T03:38:08,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-25T03:38:08,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:08,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. as already flushing 2024-11-25T03:38:08,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:08,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:08,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:08,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:08,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742108_1284 (size=12151) 2024-11-25T03:38:08,425 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/13461dee72a24f809b286de242003a4b 2024-11-25T03:38:08,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/3e64b9b076554ca59f76fcce14a6b4ec is 50, key is test_row_0/B:col10/1732505888272/Put/seqid=0 2024-11-25T03:38:08,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:08,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505948444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:08,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:08,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:08,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:08,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:08,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57870 deadline: 1732505948451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:08,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:08,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:08,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:08,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742109_1285 (size=12151) 2024-11-25T03:38:08,481 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/3e64b9b076554ca59f76fcce14a6b4ec 2024-11-25T03:38:08,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:08,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57856 deadline: 1732505948537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:08,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:08,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1732505948537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:08,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:08,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:08,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:08,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:08,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57844 deadline: 1732505948538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:08,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:08,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:08,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:08,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:08,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:08,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:08,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/ea9957fd23514cbfa0c3fdade92d487b is 50, key is test_row_0/C:col10/1732505888272/Put/seqid=0 2024-11-25T03:38:08,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:08,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505948581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:08,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:08,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:08,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:08,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:08,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57870 deadline: 1732505948581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:08,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:08,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:08,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:08,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-25T03:38:08,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:08,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. as already flushing 2024-11-25T03:38:08,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:08,673 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:08,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:08,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:08,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742110_1286 (size=12151) 2024-11-25T03:38:08,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-25T03:38:08,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:08,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. as already flushing 2024-11-25T03:38:08,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:08,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:08,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:08,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:08,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:08,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505948857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:08,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:08,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:08,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:08,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:08,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57870 deadline: 1732505948857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:08,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:08,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:08,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:09,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-25T03:38:09,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:09,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. as already flushing 2024-11-25T03:38:09,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:09,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:09,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:09,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:09,178 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/ea9957fd23514cbfa0c3fdade92d487b 2024-11-25T03:38:09,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-25T03:38:09,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:09,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. as already flushing 2024-11-25T03:38:09,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:09,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:09,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:09,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:09,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/13461dee72a24f809b286de242003a4b as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/13461dee72a24f809b286de242003a4b 2024-11-25T03:38:09,201 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/13461dee72a24f809b286de242003a4b, entries=150, sequenceid=197, filesize=11.9 K 2024-11-25T03:38:09,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/3e64b9b076554ca59f76fcce14a6b4ec as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/3e64b9b076554ca59f76fcce14a6b4ec 2024-11-25T03:38:09,206 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/3e64b9b076554ca59f76fcce14a6b4ec, entries=150, sequenceid=197, filesize=11.9 K 2024-11-25T03:38:09,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/ea9957fd23514cbfa0c3fdade92d487b as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/ea9957fd23514cbfa0c3fdade92d487b 2024-11-25T03:38:09,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/ea9957fd23514cbfa0c3fdade92d487b, entries=150, sequenceid=197, filesize=11.9 K 2024-11-25T03:38:09,218 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for c6d2702d0a1f8c4a0fdcb6498eb058d5 in 941ms, sequenceid=197, compaction requested=true 2024-11-25T03:38:09,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c6d2702d0a1f8c4a0fdcb6498eb058d5: 2024-11-25T03:38:09,219 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:38:09,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6d2702d0a1f8c4a0fdcb6498eb058d5:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:38:09,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:38:09,220 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:38:09,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6d2702d0a1f8c4a0fdcb6498eb058d5:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:38:09,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:38:09,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6d2702d0a1f8c4a0fdcb6498eb058d5:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:38:09,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:38:09,220 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:38:09,220 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): c6d2702d0a1f8c4a0fdcb6498eb058d5/A is initiating minor compaction (all files) 2024-11-25T03:38:09,220 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6d2702d0a1f8c4a0fdcb6498eb058d5/A in TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:09,220 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/e7f25cc9a0cb4586b43abd2479a1dc89, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/bc17a0bef87f484fb1eedf95a66e149b, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/13461dee72a24f809b286de242003a4b] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp, totalSize=35.9 K 2024-11-25T03:38:09,224 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7f25cc9a0cb4586b43abd2479a1dc89, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732505884470 2024-11-25T03:38:09,225 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc17a0bef87f484fb1eedf95a66e149b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732505885448 2024-11-25T03:38:09,225 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:38:09,225 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): c6d2702d0a1f8c4a0fdcb6498eb058d5/B is initiating minor compaction (all files) 2024-11-25T03:38:09,225 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6d2702d0a1f8c4a0fdcb6498eb058d5/B in TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:09,225 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/5ac60418511549b2b7a5cccbbd7a6b2f, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/b56b9ccfd2b548779dae3e7b4137b812, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/3e64b9b076554ca59f76fcce14a6b4ec] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp, totalSize=35.9 K 2024-11-25T03:38:09,226 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13461dee72a24f809b286de242003a4b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732505887132 2024-11-25T03:38:09,226 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ac60418511549b2b7a5cccbbd7a6b2f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732505884470 2024-11-25T03:38:09,226 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting b56b9ccfd2b548779dae3e7b4137b812, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732505885448 2024-11-25T03:38:09,227 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e64b9b076554ca59f76fcce14a6b4ec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732505887132 2024-11-25T03:38:09,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-25T03:38:09,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on c6d2702d0a1f8c4a0fdcb6498eb058d5 2024-11-25T03:38:09,261 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:38:09,262 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c6d2702d0a1f8c4a0fdcb6498eb058d5 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-25T03:38:09,264 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6d2702d0a1f8c4a0fdcb6498eb058d5#B#compaction#242 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:38:09,265 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/02f8d012f3c6443198c37bf8cfe3cc77 is 50, key is test_row_0/B:col10/1732505888272/Put/seqid=0 2024-11-25T03:38:09,299 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6d2702d0a1f8c4a0fdcb6498eb058d5#A#compaction#243 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:38:09,300 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/5debf147c8b1435582d96267291c8aba is 50, key is test_row_0/A:col10/1732505888272/Put/seqid=0 2024-11-25T03:38:09,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-25T03:38:09,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:09,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. as already flushing 2024-11-25T03:38:09,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:09,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:09,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:09,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:09,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/c73036f1bee64e7ebe91de68c327a073 is 50, key is test_row_0/A:col10/1732505888403/Put/seqid=0 2024-11-25T03:38:09,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742111_1287 (size=12595) 2024-11-25T03:38:09,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-25T03:38:09,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:09,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. as already flushing 2024-11-25T03:38:09,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:09,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:09,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:09,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:09,533 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/02f8d012f3c6443198c37bf8cfe3cc77 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/02f8d012f3c6443198c37bf8cfe3cc77 2024-11-25T03:38:09,541 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c6d2702d0a1f8c4a0fdcb6498eb058d5/B of c6d2702d0a1f8c4a0fdcb6498eb058d5 into 02f8d012f3c6443198c37bf8cfe3cc77(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:38:09,541 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6d2702d0a1f8c4a0fdcb6498eb058d5: 2024-11-25T03:38:09,541 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., storeName=c6d2702d0a1f8c4a0fdcb6498eb058d5/B, priority=13, startTime=1732505889219; duration=0sec 2024-11-25T03:38:09,541 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:38:09,541 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6d2702d0a1f8c4a0fdcb6498eb058d5:B 2024-11-25T03:38:09,541 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:38:09,543 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:38:09,543 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): c6d2702d0a1f8c4a0fdcb6498eb058d5/C is initiating minor compaction (all files) 2024-11-25T03:38:09,543 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6d2702d0a1f8c4a0fdcb6498eb058d5/C in TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:09,543 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/c50ea88784d94720b08b8d3b6bec91a1, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/29e2971d4ae443cc83491d0dcb62f894, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/ea9957fd23514cbfa0c3fdade92d487b] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp, totalSize=35.9 K 2024-11-25T03:38:09,543 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting c50ea88784d94720b08b8d3b6bec91a1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732505884470 2024-11-25T03:38:09,544 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 29e2971d4ae443cc83491d0dcb62f894, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732505885448 2024-11-25T03:38:09,544 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting ea9957fd23514cbfa0c3fdade92d487b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732505887132 2024-11-25T03:38:09,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742112_1288 (size=12595) 2024-11-25T03:38:09,586 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/5debf147c8b1435582d96267291c8aba as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/5debf147c8b1435582d96267291c8aba 2024-11-25T03:38:09,591 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c6d2702d0a1f8c4a0fdcb6498eb058d5/A of c6d2702d0a1f8c4a0fdcb6498eb058d5 into 5debf147c8b1435582d96267291c8aba(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:38:09,591 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6d2702d0a1f8c4a0fdcb6498eb058d5: 2024-11-25T03:38:09,592 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., storeName=c6d2702d0a1f8c4a0fdcb6498eb058d5/A, priority=13, startTime=1732505889219; duration=0sec 2024-11-25T03:38:09,592 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:38:09,592 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6d2702d0a1f8c4a0fdcb6498eb058d5:A 2024-11-25T03:38:09,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:09,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57856 deadline: 1732505949576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:09,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:09,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1732505949578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:09,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742113_1289 (size=12151) 2024-11-25T03:38:09,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:09,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:09,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:09,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:09,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:09,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:09,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:09,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57844 deadline: 1732505949586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:09,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:09,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:09,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:09,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:09,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505949591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:09,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:09,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:09,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:09,623 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6d2702d0a1f8c4a0fdcb6498eb058d5#C#compaction#245 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:38:09,624 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/21177cac61484ff5ba10734aadb3cfda is 50, key is test_row_0/C:col10/1732505888272/Put/seqid=0 2024-11-25T03:38:09,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:09,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57870 deadline: 1732505949617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:09,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:09,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:09,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:09,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742114_1290 (size=12595) 2024-11-25T03:38:09,663 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/21177cac61484ff5ba10734aadb3cfda as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/21177cac61484ff5ba10734aadb3cfda 2024-11-25T03:38:09,679 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c6d2702d0a1f8c4a0fdcb6498eb058d5/C of c6d2702d0a1f8c4a0fdcb6498eb058d5 into 21177cac61484ff5ba10734aadb3cfda(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:38:09,679 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6d2702d0a1f8c4a0fdcb6498eb058d5: 2024-11-25T03:38:09,679 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., storeName=c6d2702d0a1f8c4a0fdcb6498eb058d5/C, priority=13, startTime=1732505889220; duration=0sec 2024-11-25T03:38:09,679 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:38:09,679 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6d2702d0a1f8c4a0fdcb6498eb058d5:C 2024-11-25T03:38:09,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-25T03:38:09,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:09,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. as already flushing 2024-11-25T03:38:09,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:09,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:09,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:09,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:09,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:09,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1732505949733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:09,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:09,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57856 deadline: 1732505949733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:09,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:09,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57844 deadline: 1732505949733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:09,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:09,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57870 deadline: 1732505949737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:09,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:09,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:09,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:09,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:09,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:09,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:09,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:09,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:09,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:09,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:09,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:09,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:09,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:09,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505949753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:09,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:09,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:09,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:09,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-25T03:38:09,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:09,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. as already flushing 2024-11-25T03:38:09,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:09,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:09,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:09,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:09,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:09,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57870 deadline: 1732505949947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:09,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:09,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:09,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:09,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:09,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57844 deadline: 1732505949962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:09,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:09,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:09,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57856 deadline: 1732505949962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:09,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505949962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:09,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:09,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:09,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:09,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:09,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:09,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:09,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:09,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:09,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:09,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:09,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1732505949962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:09,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:09,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:09,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:10,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-25T03:38:10,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:10,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. as already flushing 2024-11-25T03:38:10,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:10,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:10,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:10,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:10,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/c73036f1bee64e7ebe91de68c327a073 2024-11-25T03:38:10,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/470dc5ce4424409690bf20ff57599044 is 50, key is test_row_0/B:col10/1732505888403/Put/seqid=0 2024-11-25T03:38:10,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742115_1291 (size=12151) 2024-11-25T03:38:10,038 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/470dc5ce4424409690bf20ff57599044 2024-11-25T03:38:10,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/a08a68bbfa8e461f9beae6916d8fbf79 is 50, key is test_row_0/C:col10/1732505888403/Put/seqid=0 2024-11-25T03:38:10,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742116_1292 (size=12151) 2024-11-25T03:38:10,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-25T03:38:10,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:10,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. as already flushing 2024-11-25T03:38:10,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:10,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:10,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:10,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:10,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:10,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57870 deadline: 1732505950267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:10,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:10,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:10,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:10,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:10,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57856 deadline: 1732505950278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:10,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:10,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505950278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:10,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:10,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:10,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:10,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:10,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:10,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:10,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1732505950278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:10,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:10,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:10,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:10,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:10,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:10,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57844 deadline: 1732505950278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:10,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:10,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:10,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:10,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-25T03:38:10,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:10,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. as already flushing 2024-11-25T03:38:10,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:10,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:10,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:10,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:10,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-25T03:38:10,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:10,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. as already flushing 2024-11-25T03:38:10,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:10,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:10,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:10,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:10,480 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/a08a68bbfa8e461f9beae6916d8fbf79 2024-11-25T03:38:10,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/c73036f1bee64e7ebe91de68c327a073 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/c73036f1bee64e7ebe91de68c327a073 2024-11-25T03:38:10,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/c73036f1bee64e7ebe91de68c327a073, entries=150, sequenceid=209, filesize=11.9 K 2024-11-25T03:38:10,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/470dc5ce4424409690bf20ff57599044 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/470dc5ce4424409690bf20ff57599044 2024-11-25T03:38:10,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/470dc5ce4424409690bf20ff57599044, entries=150, sequenceid=209, filesize=11.9 K 2024-11-25T03:38:10,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/a08a68bbfa8e461f9beae6916d8fbf79 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/a08a68bbfa8e461f9beae6916d8fbf79 2024-11-25T03:38:10,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/a08a68bbfa8e461f9beae6916d8fbf79, entries=150, sequenceid=209, filesize=11.9 K 2024-11-25T03:38:10,501 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for c6d2702d0a1f8c4a0fdcb6498eb058d5 in 1239ms, sequenceid=209, compaction requested=false 2024-11-25T03:38:10,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c6d2702d0a1f8c4a0fdcb6498eb058d5: 2024-11-25T03:38:10,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-25T03:38:10,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:10,625 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing c6d2702d0a1f8c4a0fdcb6498eb058d5 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-25T03:38:10,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/68a65058743e4e6e8c2f10ab73f254ab is 50, key is test_row_0/A:col10/1732505889613/Put/seqid=0 2024-11-25T03:38:10,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742117_1293 (size=12151) 2024-11-25T03:38:10,640 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/68a65058743e4e6e8c2f10ab73f254ab 2024-11-25T03:38:10,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/87d4adb896a945cabe4f620fb8ea11f7 is 50, key is test_row_0/B:col10/1732505889613/Put/seqid=0 2024-11-25T03:38:10,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742118_1294 (size=12151) 2024-11-25T03:38:10,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on c6d2702d0a1f8c4a0fdcb6498eb058d5 2024-11-25T03:38:10,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. as already flushing 2024-11-25T03:38:10,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:10,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57856 deadline: 1732505950793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:10,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:10,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505950793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:10,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:10,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:10,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:10,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:10,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:10,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:10,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:10,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1732505950794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:10,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:10,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57870 deadline: 1732505950794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:10,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:10,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:10,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:10,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:10,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:10,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:10,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:10,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57844 deadline: 1732505950797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:10,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:10,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:10,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:10,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:10,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57856 deadline: 1732505950907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:10,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:10,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:10,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:10,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505950908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:10,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57870 deadline: 1732505950908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:10,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:10,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:10,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:10,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:10,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:10,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:10,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:10,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1732505950908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:10,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:10,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:10,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:10,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57844 deadline: 1732505950908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:10,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:10,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:10,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:10,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:10,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:10,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:11,054 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/87d4adb896a945cabe4f620fb8ea11f7 2024-11-25T03:38:11,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/984153367ac5446a888b9c138fcb7f70 is 50, key is test_row_0/C:col10/1732505889613/Put/seqid=0 2024-11-25T03:38:11,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742119_1295 (size=12151) 2024-11-25T03:38:11,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:11,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:11,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505951118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:11,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57856 deadline: 1732505951118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:11,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:11,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:11,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57870 deadline: 1732505951118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:11,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:11,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:11,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:11,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1732505951118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:11,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:11,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:11,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57844 deadline: 1732505951118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:11,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:11,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:11,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:11,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:11,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:11,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:11,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:11,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:11,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:11,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:11,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:11,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-25T03:38:11,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:11,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:11,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505951427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:11,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57856 deadline: 1732505951427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:11,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:11,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:11,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:11,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:11,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:11,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57870 deadline: 1732505951427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:11,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:11,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:11,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1732505951428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:11,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:11,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:11,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:11,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:11,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:11,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57844 deadline: 1732505951428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:11,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:11,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:11,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:11,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:11,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:11,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:11,467 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/984153367ac5446a888b9c138fcb7f70 2024-11-25T03:38:11,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/68a65058743e4e6e8c2f10ab73f254ab as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/68a65058743e4e6e8c2f10ab73f254ab 2024-11-25T03:38:11,476 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/68a65058743e4e6e8c2f10ab73f254ab, entries=150, sequenceid=236, filesize=11.9 K 2024-11-25T03:38:11,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/87d4adb896a945cabe4f620fb8ea11f7 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/87d4adb896a945cabe4f620fb8ea11f7 2024-11-25T03:38:11,480 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/87d4adb896a945cabe4f620fb8ea11f7, entries=150, sequenceid=236, filesize=11.9 K 2024-11-25T03:38:11,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/984153367ac5446a888b9c138fcb7f70 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/984153367ac5446a888b9c138fcb7f70 2024-11-25T03:38:11,484 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/984153367ac5446a888b9c138fcb7f70, entries=150, sequenceid=236, filesize=11.9 K 2024-11-25T03:38:11,485 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for c6d2702d0a1f8c4a0fdcb6498eb058d5 in 860ms, sequenceid=236, compaction requested=true 2024-11-25T03:38:11,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for c6d2702d0a1f8c4a0fdcb6498eb058d5: 2024-11-25T03:38:11,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:11,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-25T03:38:11,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-25T03:38:11,488 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-25T03:38:11,488 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.4630 sec 2024-11-25T03:38:11,490 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 4.4710 sec 2024-11-25T03:38:11,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on c6d2702d0a1f8c4a0fdcb6498eb058d5 2024-11-25T03:38:11,940 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:38:11,940 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c6d2702d0a1f8c4a0fdcb6498eb058d5 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-25T03:38:11,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/53d4ac435d7d413d8abd396ae66e6055 is 50, key is test_row_0/A:col10/1732505890792/Put/seqid=0 2024-11-25T03:38:11,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742120_1296 (size=12151) 2024-11-25T03:38:11,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:11,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:11,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505951964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:11,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57870 deadline: 1732505951963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:11,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:11,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:11,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:11,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:11,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:11,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:11,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:11,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57856 deadline: 1732505951969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:11,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:11,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:11,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:11,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:11,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:11,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57844 deadline: 1732505951970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:11,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1732505951970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:11,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:11,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:11,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:11,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:11,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:11,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:12,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:12,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505952077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:12,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:12,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:12,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:12,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57870 deadline: 1732505952077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:12,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:12,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:12,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:12,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57856 deadline: 1732505952078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:12,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:12,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:12,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:12,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:12,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:12,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:12,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57844 deadline: 1732505952087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:12,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:12,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:12,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:12,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:12,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1732505952087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:12,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:12,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:12,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:12,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:12,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505952287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:12,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:12,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57870 deadline: 1732505952287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:12,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:12,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:12,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:12,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:12,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57856 deadline: 1732505952287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:12,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:12,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:12,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:12,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:12,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:12,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:12,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:12,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57844 deadline: 1732505952297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:12,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:12,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:12,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:12,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1732505952297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:12,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:12,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:12,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:12,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:12,351 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/53d4ac435d7d413d8abd396ae66e6055 2024-11-25T03:38:12,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/6002a427a0c14c41a57a9df0eca3123c is 50, key is test_row_0/B:col10/1732505890792/Put/seqid=0 2024-11-25T03:38:12,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742121_1297 (size=12151) 2024-11-25T03:38:12,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:12,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505952597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:12,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:12,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:12,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:12,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57870 deadline: 1732505952597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:12,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:12,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:12,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:12,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:12,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:12,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57856 deadline: 1732505952597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:12,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:12,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:12,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:12,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:12,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57844 deadline: 1732505952608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:12,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:12,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1732505952608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:12,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:12,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:12,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:12,611 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:12,611 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:12,611 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:12,763 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/6002a427a0c14c41a57a9df0eca3123c 2024-11-25T03:38:12,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/2f968a32b6ab4f7daf7de1cd37d585d5 is 50, key is test_row_0/C:col10/1732505890792/Put/seqid=0 2024-11-25T03:38:12,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742122_1298 (size=12151) 2024-11-25T03:38:13,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:13,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505953107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:13,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:13,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:13,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:13,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:13,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57870 deadline: 1732505953108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:13,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:13,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57856 deadline: 1732505953108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:13,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:13,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:13,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:13,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:13,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:13,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:13,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:13,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57844 deadline: 1732505953117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:13,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:13,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:13,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:13,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1732505953117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:13,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:13,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:13,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:13,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:13,179 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/2f968a32b6ab4f7daf7de1cd37d585d5 2024-11-25T03:38:13,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/53d4ac435d7d413d8abd396ae66e6055 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/53d4ac435d7d413d8abd396ae66e6055 2024-11-25T03:38:13,187 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/53d4ac435d7d413d8abd396ae66e6055, entries=150, sequenceid=250, filesize=11.9 K 2024-11-25T03:38:13,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/6002a427a0c14c41a57a9df0eca3123c as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/6002a427a0c14c41a57a9df0eca3123c 2024-11-25T03:38:13,192 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/6002a427a0c14c41a57a9df0eca3123c, entries=150, sequenceid=250, filesize=11.9 K 2024-11-25T03:38:13,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/2f968a32b6ab4f7daf7de1cd37d585d5 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/2f968a32b6ab4f7daf7de1cd37d585d5 2024-11-25T03:38:13,196 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/2f968a32b6ab4f7daf7de1cd37d585d5, entries=150, sequenceid=250, filesize=11.9 K 2024-11-25T03:38:13,197 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for c6d2702d0a1f8c4a0fdcb6498eb058d5 in 1256ms, sequenceid=250, compaction requested=true 2024-11-25T03:38:13,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c6d2702d0a1f8c4a0fdcb6498eb058d5: 2024-11-25T03:38:13,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6d2702d0a1f8c4a0fdcb6498eb058d5:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:38:13,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:38:13,197 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:38:13,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6d2702d0a1f8c4a0fdcb6498eb058d5:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:38:13,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:38:13,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6d2702d0a1f8c4a0fdcb6498eb058d5:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:38:13,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:38:13,197 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:38:13,200 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:38:13,200 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:38:13,200 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): c6d2702d0a1f8c4a0fdcb6498eb058d5/A is initiating minor compaction (all files) 2024-11-25T03:38:13,200 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): c6d2702d0a1f8c4a0fdcb6498eb058d5/B is initiating minor compaction (all files) 2024-11-25T03:38:13,200 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6d2702d0a1f8c4a0fdcb6498eb058d5/A in TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:13,200 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6d2702d0a1f8c4a0fdcb6498eb058d5/B in TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:13,200 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/5debf147c8b1435582d96267291c8aba, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/c73036f1bee64e7ebe91de68c327a073, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/68a65058743e4e6e8c2f10ab73f254ab, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/53d4ac435d7d413d8abd396ae66e6055] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp, totalSize=47.9 K 2024-11-25T03:38:13,200 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/02f8d012f3c6443198c37bf8cfe3cc77, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/470dc5ce4424409690bf20ff57599044, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/87d4adb896a945cabe4f620fb8ea11f7, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/6002a427a0c14c41a57a9df0eca3123c] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp, totalSize=47.9 K 2024-11-25T03:38:13,200 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5debf147c8b1435582d96267291c8aba, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732505887132 2024-11-25T03:38:13,200 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 02f8d012f3c6443198c37bf8cfe3cc77, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732505887132 2024-11-25T03:38:13,201 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 470dc5ce4424409690bf20ff57599044, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732505888315 2024-11-25T03:38:13,201 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting c73036f1bee64e7ebe91de68c327a073, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732505888315 2024-11-25T03:38:13,201 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 87d4adb896a945cabe4f620fb8ea11f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732505889538 2024-11-25T03:38:13,201 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68a65058743e4e6e8c2f10ab73f254ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732505889538 2024-11-25T03:38:13,201 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 6002a427a0c14c41a57a9df0eca3123c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732505890792 2024-11-25T03:38:13,201 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53d4ac435d7d413d8abd396ae66e6055, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732505890792 2024-11-25T03:38:13,212 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6d2702d0a1f8c4a0fdcb6498eb058d5#A#compaction#254 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:38:13,213 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/a264cb364d024e7d9b60aa06adc9ed63 is 50, key is test_row_0/A:col10/1732505890792/Put/seqid=0 2024-11-25T03:38:13,219 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6d2702d0a1f8c4a0fdcb6498eb058d5#B#compaction#255 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:38:13,219 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/2bf751a1bd3a42b190a5cc7ecac648b4 is 50, key is test_row_0/B:col10/1732505890792/Put/seqid=0 2024-11-25T03:38:13,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742123_1299 (size=12731) 2024-11-25T03:38:13,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742124_1300 (size=12731) 2024-11-25T03:38:13,228 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/a264cb364d024e7d9b60aa06adc9ed63 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/a264cb364d024e7d9b60aa06adc9ed63 2024-11-25T03:38:13,233 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c6d2702d0a1f8c4a0fdcb6498eb058d5/A of c6d2702d0a1f8c4a0fdcb6498eb058d5 into a264cb364d024e7d9b60aa06adc9ed63(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:38:13,233 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6d2702d0a1f8c4a0fdcb6498eb058d5: 2024-11-25T03:38:13,233 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., storeName=c6d2702d0a1f8c4a0fdcb6498eb058d5/A, priority=12, startTime=1732505893197; duration=0sec 2024-11-25T03:38:13,234 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:38:13,234 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6d2702d0a1f8c4a0fdcb6498eb058d5:A 2024-11-25T03:38:13,234 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:38:13,235 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:38:13,235 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): c6d2702d0a1f8c4a0fdcb6498eb058d5/C is initiating minor compaction (all files) 2024-11-25T03:38:13,236 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6d2702d0a1f8c4a0fdcb6498eb058d5/C in TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:13,236 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/21177cac61484ff5ba10734aadb3cfda, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/a08a68bbfa8e461f9beae6916d8fbf79, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/984153367ac5446a888b9c138fcb7f70, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/2f968a32b6ab4f7daf7de1cd37d585d5] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp, totalSize=47.9 K 2024-11-25T03:38:13,236 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21177cac61484ff5ba10734aadb3cfda, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732505887132 2024-11-25T03:38:13,236 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting a08a68bbfa8e461f9beae6916d8fbf79, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732505888315 2024-11-25T03:38:13,237 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 984153367ac5446a888b9c138fcb7f70, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732505889538 2024-11-25T03:38:13,237 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f968a32b6ab4f7daf7de1cd37d585d5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732505890792 2024-11-25T03:38:13,247 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6d2702d0a1f8c4a0fdcb6498eb058d5#C#compaction#256 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:38:13,247 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/c4e8bee812a6485f9dc974c5c3bed1ef is 50, key is test_row_0/C:col10/1732505890792/Put/seqid=0 2024-11-25T03:38:13,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742125_1301 (size=12731) 2024-11-25T03:38:13,632 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/2bf751a1bd3a42b190a5cc7ecac648b4 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/2bf751a1bd3a42b190a5cc7ecac648b4 2024-11-25T03:38:13,637 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c6d2702d0a1f8c4a0fdcb6498eb058d5/B of c6d2702d0a1f8c4a0fdcb6498eb058d5 into 2bf751a1bd3a42b190a5cc7ecac648b4(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:38:13,637 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6d2702d0a1f8c4a0fdcb6498eb058d5: 2024-11-25T03:38:13,637 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., storeName=c6d2702d0a1f8c4a0fdcb6498eb058d5/B, priority=12, startTime=1732505893197; duration=0sec 2024-11-25T03:38:13,637 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:38:13,637 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6d2702d0a1f8c4a0fdcb6498eb058d5:B 2024-11-25T03:38:13,667 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/c4e8bee812a6485f9dc974c5c3bed1ef as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/c4e8bee812a6485f9dc974c5c3bed1ef 2024-11-25T03:38:13,672 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c6d2702d0a1f8c4a0fdcb6498eb058d5/C of c6d2702d0a1f8c4a0fdcb6498eb058d5 into c4e8bee812a6485f9dc974c5c3bed1ef(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:38:13,672 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6d2702d0a1f8c4a0fdcb6498eb058d5: 2024-11-25T03:38:13,672 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., storeName=c6d2702d0a1f8c4a0fdcb6498eb058d5/C, priority=12, startTime=1732505893197; duration=0sec 2024-11-25T03:38:13,672 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:38:13,672 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6d2702d0a1f8c4a0fdcb6498eb058d5:C 2024-11-25T03:38:14,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on c6d2702d0a1f8c4a0fdcb6498eb058d5 2024-11-25T03:38:14,120 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:38:14,120 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c6d2702d0a1f8c4a0fdcb6498eb058d5 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-25T03:38:14,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/d23bb190567344c59cafba7ec6c934c5 is 50, key is test_row_0/A:col10/1732505894119/Put/seqid=0 2024-11-25T03:38:14,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742126_1302 (size=12301) 2024-11-25T03:38:14,131 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/d23bb190567344c59cafba7ec6c934c5 2024-11-25T03:38:14,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:14,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57870 deadline: 1732505954134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:14,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:14,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:14,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:14,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/da6e41961c654033a4f81eda2d84c3b4 is 50, key is test_row_0/B:col10/1732505894119/Put/seqid=0 2024-11-25T03:38:14,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:14,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505954135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:14,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:14,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:14,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:14,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:14,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57856 deadline: 1732505954136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:14,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:14,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:14,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:14,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:14,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57844 deadline: 1732505954138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:14,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:14,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1732505954138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:14,141 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,34881,1732505806588 for 'test_row_1' in c6d2702d0a1f8c4a0fdcb6498eb058d5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:38:14,141 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,34881,1732505806588 for 'test_row_2' in c6d2702d0a1f8c4a0fdcb6498eb058d5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:38:14,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:14,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:14,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:14,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:14,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:14,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:14,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742127_1303 (size=12301) 2024-11-25T03:38:14,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:14,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:14,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505954248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:14,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57870 deadline: 1732505954248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:14,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:14,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:14,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:14,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57856 deadline: 1732505954248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:14,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:14,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:14,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:14,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:14,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:14,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:14,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:14,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:14,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57870 deadline: 1732505954457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:14,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:14,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:14,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:14,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:14,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505954457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:14,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:14,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:14,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:14,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:14,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57856 deadline: 1732505954457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:14,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:14,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:14,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:14,544 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/da6e41961c654033a4f81eda2d84c3b4 2024-11-25T03:38:14,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/8964a75802174316ba7850736d416d9b is 50, key is test_row_0/C:col10/1732505894119/Put/seqid=0 2024-11-25T03:38:14,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742128_1304 (size=12301) 2024-11-25T03:38:14,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:14,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57870 deadline: 1732505954768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:14,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:14,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505954768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:14,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:14,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:14,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57856 deadline: 1732505954768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:14,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:14,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:14,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:14,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:14,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:14,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:14,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:14,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:14,798 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T03:38:14,956 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/8964a75802174316ba7850736d416d9b 2024-11-25T03:38:14,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/d23bb190567344c59cafba7ec6c934c5 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/d23bb190567344c59cafba7ec6c934c5 2024-11-25T03:38:14,964 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/d23bb190567344c59cafba7ec6c934c5, entries=150, sequenceid=278, filesize=12.0 K 2024-11-25T03:38:14,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/da6e41961c654033a4f81eda2d84c3b4 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/da6e41961c654033a4f81eda2d84c3b4 2024-11-25T03:38:14,968 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/da6e41961c654033a4f81eda2d84c3b4, entries=150, sequenceid=278, filesize=12.0 K 2024-11-25T03:38:14,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/8964a75802174316ba7850736d416d9b as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/8964a75802174316ba7850736d416d9b 2024-11-25T03:38:14,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/8964a75802174316ba7850736d416d9b, entries=150, sequenceid=278, filesize=12.0 K 2024-11-25T03:38:14,974 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for c6d2702d0a1f8c4a0fdcb6498eb058d5 in 854ms, sequenceid=278, compaction requested=false 2024-11-25T03:38:14,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c6d2702d0a1f8c4a0fdcb6498eb058d5: 2024-11-25T03:38:15,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-25T03:38:15,278 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:38:15,278 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:38:15,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-25T03:38:15,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on c6d2702d0a1f8c4a0fdcb6498eb058d5 2024-11-25T03:38:15,280 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:38:15,280 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c6d2702d0a1f8c4a0fdcb6498eb058d5 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-25T03:38:15,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-25T03:38:15,281 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:38:15,281 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:38:15,282 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:38:15,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/a5b457cf672a4c659b100937ed670163 is 50, key is test_row_0/A:col10/1732505895279/Put/seqid=0 2024-11-25T03:38:15,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742129_1305 (size=14741) 2024-11-25T03:38:15,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:15,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57870 deadline: 1732505955309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:15,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:15,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:15,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:15,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57856 deadline: 1732505955309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:15,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:15,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:15,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:15,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:15,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:15,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505955312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:15,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:15,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:15,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:15,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-25T03:38:15,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:15,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57870 deadline: 1732505955417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:15,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:15,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57856 deadline: 1732505955417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:15,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:15,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:15,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:15,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:15,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:15,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:15,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:15,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505955427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:15,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:15,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:15,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:15,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-25T03:38:15,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:15,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. as already flushing 2024-11-25T03:38:15,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:15,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:15,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:15,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:15,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-25T03:38:15,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:15,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. as already flushing 2024-11-25T03:38:15,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:15,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:15,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:15,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:15,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-25T03:38:15,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:15,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57870 deadline: 1732505955627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:15,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:15,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:15,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:15,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57856 deadline: 1732505955627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:15,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:15,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:15,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:15,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:15,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:15,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505955637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:15,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:15,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:15,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:15,690 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/a5b457cf672a4c659b100937ed670163 2024-11-25T03:38:15,697 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/9416cf1990b74663881d2079c0ed2b96 is 50, key is test_row_0/B:col10/1732505895279/Put/seqid=0 2024-11-25T03:38:15,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742130_1306 (size=12301) 2024-11-25T03:38:15,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-25T03:38:15,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:15,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. as already flushing 2024-11-25T03:38:15,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:15,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:15,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:15,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:15,795 INFO [Thread-1163 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1163. 2024-11-25T03:38:15,795 DEBUG [Thread-1163 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:38:15,795 DEBUG [Thread-1163 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:15,795 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:38:15,795 DEBUG [Thread-1163 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:15,796 INFO [Thread-1159 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1159. 2024-11-25T03:38:15,796 DEBUG [Thread-1159 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:38:15,796 DEBUG [Thread-1159 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:15,796 DEBUG [Thread-1159 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:15,796 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:38:15,797 INFO [Thread-1167 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1167. 2024-11-25T03:38:15,797 DEBUG [Thread-1167 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:38:15,797 DEBUG [Thread-1167 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:15,797 DEBUG [Thread-1167 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:15,797 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:38:15,799 INFO [Thread-1165 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1165. 2024-11-25T03:38:15,799 DEBUG [Thread-1165 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:38:15,799 DEBUG [Thread-1165 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:15,799 INFO [Thread-1161 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1161. 2024-11-25T03:38:15,799 DEBUG [Thread-1161 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:38:15,799 DEBUG [Thread-1165 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:15,799 DEBUG [Thread-1161 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:15,799 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:38:15,800 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:38:15,800 DEBUG [Thread-1161 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:15,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-25T03:38:15,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:15,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. as already flushing 2024-11-25T03:38:15,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:15,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:15,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:15,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:15,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-25T03:38:15,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:15,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:15,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57856 deadline: 1732505955937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:15,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57870 deadline: 1732505955937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:15,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:15,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:15,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:15,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:15,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:15,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:15,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:15,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505955948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:15,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:15,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:15,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:16,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-25T03:38:16,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:16,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. as already flushing 2024-11-25T03:38:16,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:16,046 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:16,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:16,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:16,101 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/9416cf1990b74663881d2079c0ed2b96 2024-11-25T03:38:16,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/aadccdfb5554497492d50f1d5012c36b is 50, key is test_row_0/C:col10/1732505895279/Put/seqid=0 2024-11-25T03:38:16,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742131_1307 (size=12301) 2024-11-25T03:38:16,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:16,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57804 deadline: 1732505956147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:16,148 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,34881,1732505806588 for 'test_row_2' in c6d2702d0a1f8c4a0fdcb6498eb058d5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:38:16,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:16,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:16,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:16,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:16,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57844 deadline: 1732505956157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:16,158 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,34881,1732505806588 for 'test_row_1' in c6d2702d0a1f8c4a0fdcb6498eb058d5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:38:16,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:16,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:16,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:16,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-25T03:38:16,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:16,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. as already flushing 2024-11-25T03:38:16,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:16,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:16,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:16,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:16,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-25T03:38:16,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:16,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. as already flushing 2024-11-25T03:38:16,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:16,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:16,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:16,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:16,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-25T03:38:16,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:16,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:16,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57870 deadline: 1732505956448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:16,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57856 deadline: 1732505956448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:16,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:16,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:16,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:16,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:16,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:16,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:16,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:16,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57872 deadline: 1732505956457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:16,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:16,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:16,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:16,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-25T03:38:16,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:16,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. as already flushing 2024-11-25T03:38:16,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:16,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:16,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:16,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:16,511 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/aadccdfb5554497492d50f1d5012c36b 2024-11-25T03:38:16,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/a5b457cf672a4c659b100937ed670163 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/a5b457cf672a4c659b100937ed670163 2024-11-25T03:38:16,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/a5b457cf672a4c659b100937ed670163, entries=200, sequenceid=292, filesize=14.4 K 2024-11-25T03:38:16,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/9416cf1990b74663881d2079c0ed2b96 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/9416cf1990b74663881d2079c0ed2b96 2024-11-25T03:38:16,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/9416cf1990b74663881d2079c0ed2b96, entries=150, sequenceid=292, filesize=12.0 K 2024-11-25T03:38:16,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/aadccdfb5554497492d50f1d5012c36b as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/aadccdfb5554497492d50f1d5012c36b 2024-11-25T03:38:16,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/aadccdfb5554497492d50f1d5012c36b, entries=150, sequenceid=292, filesize=12.0 K 2024-11-25T03:38:16,526 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for c6d2702d0a1f8c4a0fdcb6498eb058d5 in 1246ms, sequenceid=292, compaction requested=true 2024-11-25T03:38:16,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c6d2702d0a1f8c4a0fdcb6498eb058d5: 2024-11-25T03:38:16,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6d2702d0a1f8c4a0fdcb6498eb058d5:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:38:16,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:38:16,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6d2702d0a1f8c4a0fdcb6498eb058d5:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:38:16,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:38:16,527 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:38:16,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c6d2702d0a1f8c4a0fdcb6498eb058d5:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:38:16,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:38:16,527 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:38:16,527 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39773 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:38:16,527 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:38:16,527 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): c6d2702d0a1f8c4a0fdcb6498eb058d5/A is initiating minor compaction (all files) 2024-11-25T03:38:16,527 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): c6d2702d0a1f8c4a0fdcb6498eb058d5/B is initiating minor compaction (all files) 2024-11-25T03:38:16,527 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6d2702d0a1f8c4a0fdcb6498eb058d5/A in TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:16,527 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6d2702d0a1f8c4a0fdcb6498eb058d5/B in TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:16,528 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/a264cb364d024e7d9b60aa06adc9ed63, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/d23bb190567344c59cafba7ec6c934c5, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/a5b457cf672a4c659b100937ed670163] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp, totalSize=38.8 K 2024-11-25T03:38:16,528 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/2bf751a1bd3a42b190a5cc7ecac648b4, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/da6e41961c654033a4f81eda2d84c3b4, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/9416cf1990b74663881d2079c0ed2b96] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp, totalSize=36.5 K 2024-11-25T03:38:16,528 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting a264cb364d024e7d9b60aa06adc9ed63, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732505890792 2024-11-25T03:38:16,528 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bf751a1bd3a42b190a5cc7ecac648b4, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732505890792 2024-11-25T03:38:16,528 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting da6e41961c654033a4f81eda2d84c3b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732505891969 2024-11-25T03:38:16,528 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting d23bb190567344c59cafba7ec6c934c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732505891969 2024-11-25T03:38:16,528 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5b457cf672a4c659b100937ed670163, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732505894132 2024-11-25T03:38:16,528 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 9416cf1990b74663881d2079c0ed2b96, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732505894132 2024-11-25T03:38:16,534 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6d2702d0a1f8c4a0fdcb6498eb058d5#B#compaction#263 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:38:16,534 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/2dd8dd54e9cb45829a5f566b0a15909e is 50, key is test_row_0/B:col10/1732505895279/Put/seqid=0 2024-11-25T03:38:16,535 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6d2702d0a1f8c4a0fdcb6498eb058d5#A#compaction#264 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:38:16,536 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/572f4686fb9e4ad28990301956e9f6fc is 50, key is test_row_0/A:col10/1732505895279/Put/seqid=0 2024-11-25T03:38:16,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742132_1308 (size=12983) 2024-11-25T03:38:16,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742133_1309 (size=12983) 2024-11-25T03:38:16,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-25T03:38:16,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:16,658 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing c6d2702d0a1f8c4a0fdcb6498eb058d5 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-25T03:38:16,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/9c2159d85ea441a4850869b766159226 is 50, key is test_row_0/A:col10/1732505895308/Put/seqid=0 2024-11-25T03:38:16,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742134_1310 (size=12301) 2024-11-25T03:38:16,942 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/2dd8dd54e9cb45829a5f566b0a15909e as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/2dd8dd54e9cb45829a5f566b0a15909e 2024-11-25T03:38:16,943 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/572f4686fb9e4ad28990301956e9f6fc as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/572f4686fb9e4ad28990301956e9f6fc 2024-11-25T03:38:16,946 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c6d2702d0a1f8c4a0fdcb6498eb058d5/B of c6d2702d0a1f8c4a0fdcb6498eb058d5 into 2dd8dd54e9cb45829a5f566b0a15909e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:38:16,946 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6d2702d0a1f8c4a0fdcb6498eb058d5: 2024-11-25T03:38:16,946 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., storeName=c6d2702d0a1f8c4a0fdcb6498eb058d5/B, priority=13, startTime=1732505896526; duration=0sec 2024-11-25T03:38:16,946 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c6d2702d0a1f8c4a0fdcb6498eb058d5/A of c6d2702d0a1f8c4a0fdcb6498eb058d5 into 572f4686fb9e4ad28990301956e9f6fc(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:38:16,946 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:38:16,946 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6d2702d0a1f8c4a0fdcb6498eb058d5: 2024-11-25T03:38:16,946 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6d2702d0a1f8c4a0fdcb6498eb058d5:B 2024-11-25T03:38:16,946 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., storeName=c6d2702d0a1f8c4a0fdcb6498eb058d5/A, priority=13, startTime=1732505896526; duration=0sec 2024-11-25T03:38:16,947 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:38:16,947 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:38:16,947 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6d2702d0a1f8c4a0fdcb6498eb058d5:A 2024-11-25T03:38:16,947 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:38:16,947 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): c6d2702d0a1f8c4a0fdcb6498eb058d5/C is initiating minor compaction (all files) 2024-11-25T03:38:16,947 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c6d2702d0a1f8c4a0fdcb6498eb058d5/C in TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:16,947 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/c4e8bee812a6485f9dc974c5c3bed1ef, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/8964a75802174316ba7850736d416d9b, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/aadccdfb5554497492d50f1d5012c36b] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp, totalSize=36.5 K 2024-11-25T03:38:16,948 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting c4e8bee812a6485f9dc974c5c3bed1ef, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732505890792 2024-11-25T03:38:16,948 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 8964a75802174316ba7850736d416d9b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732505891969 2024-11-25T03:38:16,948 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting aadccdfb5554497492d50f1d5012c36b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732505894132 2024-11-25T03:38:16,953 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c6d2702d0a1f8c4a0fdcb6498eb058d5#C#compaction#266 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:38:16,954 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/af240d7aaa654b679e66c499b5a6eee0 is 50, key is test_row_0/C:col10/1732505895279/Put/seqid=0 2024-11-25T03:38:16,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742135_1311 (size=12983) 2024-11-25T03:38:17,066 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/9c2159d85ea441a4850869b766159226 2024-11-25T03:38:17,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/d590cbcb85064a4bb6cab27979aacf37 is 50, key is test_row_0/B:col10/1732505895308/Put/seqid=0 2024-11-25T03:38:17,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742136_1312 (size=12301) 2024-11-25T03:38:17,361 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/af240d7aaa654b679e66c499b5a6eee0 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/af240d7aaa654b679e66c499b5a6eee0 2024-11-25T03:38:17,366 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c6d2702d0a1f8c4a0fdcb6498eb058d5/C of c6d2702d0a1f8c4a0fdcb6498eb058d5 into af240d7aaa654b679e66c499b5a6eee0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:38:17,366 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c6d2702d0a1f8c4a0fdcb6498eb058d5: 2024-11-25T03:38:17,366 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., storeName=c6d2702d0a1f8c4a0fdcb6498eb058d5/C, priority=13, startTime=1732505896527; duration=0sec 2024-11-25T03:38:17,366 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:38:17,366 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c6d2702d0a1f8c4a0fdcb6498eb058d5:C 2024-11-25T03:38:17,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-25T03:38:17,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on c6d2702d0a1f8c4a0fdcb6498eb058d5 2024-11-25T03:38:17,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. as already flushing 2024-11-25T03:38:17,459 INFO [Thread-1156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1156. 2024-11-25T03:38:17,459 INFO [Thread-1150 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1150. 2024-11-25T03:38:17,459 DEBUG [Thread-1156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:38:17,459 DEBUG [Thread-1150 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:38:17,459 DEBUG [Thread-1156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:17,459 DEBUG [Thread-1150 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:17,459 DEBUG [Thread-1156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:17,459 DEBUG [Thread-1150 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:17,459 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:38:17,459 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:38:17,469 INFO [Thread-1154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1154. 2024-11-25T03:38:17,469 DEBUG [Thread-1154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:38:17,469 DEBUG [Thread-1154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:17,469 DEBUG [Thread-1154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:17,469 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:38:17,476 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/d590cbcb85064a4bb6cab27979aacf37 2024-11-25T03:38:17,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/12d38c447ad94e9884a0f43870a20ba4 is 50, key is test_row_0/C:col10/1732505895308/Put/seqid=0 2024-11-25T03:38:17,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742137_1313 (size=12301) 2024-11-25T03:38:17,886 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/12d38c447ad94e9884a0f43870a20ba4 2024-11-25T03:38:17,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/9c2159d85ea441a4850869b766159226 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/9c2159d85ea441a4850869b766159226 2024-11-25T03:38:17,894 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/9c2159d85ea441a4850869b766159226, entries=150, sequenceid=314, filesize=12.0 K 2024-11-25T03:38:17,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/d590cbcb85064a4bb6cab27979aacf37 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/d590cbcb85064a4bb6cab27979aacf37 2024-11-25T03:38:17,898 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/d590cbcb85064a4bb6cab27979aacf37, entries=150, sequenceid=314, filesize=12.0 K 2024-11-25T03:38:17,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/12d38c447ad94e9884a0f43870a20ba4 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/12d38c447ad94e9884a0f43870a20ba4 2024-11-25T03:38:17,901 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/12d38c447ad94e9884a0f43870a20ba4, entries=150, sequenceid=314, filesize=12.0 K 2024-11-25T03:38:17,902 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=20.13 KB/20610 for c6d2702d0a1f8c4a0fdcb6498eb058d5 in 1244ms, sequenceid=314, compaction requested=false 2024-11-25T03:38:17,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for c6d2702d0a1f8c4a0fdcb6498eb058d5: 2024-11-25T03:38:17,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:17,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-25T03:38:17,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-25T03:38:17,904 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-25T03:38:17,905 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6210 sec 2024-11-25T03:38:17,906 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 2.6270 sec 2024-11-25T03:38:19,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-25T03:38:19,447 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:38:20,199 INFO [Thread-1152 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1152. 2024-11-25T03:38:20,199 INFO [Thread-1148 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1148. 2024-11-25T03:38:20,199 DEBUG [Thread-1152 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:38:20,199 DEBUG [Thread-1148 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:38:20,199 DEBUG [Thread-1152 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:20,199 DEBUG [Thread-1148 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:20,200 DEBUG [Thread-1148 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:20,200 DEBUG [Thread-1152 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:20,200 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:38:20,200 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-25T03:38:20,200 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-11-25T03:38:20,200 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-11-25T03:38:20,200 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-25T03:38:20,200 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-25T03:38:20,200 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-25T03:38:20,200 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-25T03:38:20,200 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1450 2024-11-25T03:38:20,200 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:38:20,200 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1440 2024-11-25T03:38:20,200 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1359 2024-11-25T03:38:20,200 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1432 2024-11-25T03:38:20,200 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1423 2024-11-25T03:38:20,200 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-25T03:38:20,200 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T03:38:20,200 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T03:38:20,200 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:20,200 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:20,200 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:38:20,201 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-25T03:38:20,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-25T03:38:20,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-25T03:38:20,204 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732505900204"}]},"ts":"1732505900204"} 2024-11-25T03:38:20,205 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-25T03:38:20,205 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-25T03:38:20,206 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T03:38:20,207 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c6d2702d0a1f8c4a0fdcb6498eb058d5, UNASSIGN}] 2024-11-25T03:38:20,208 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c6d2702d0a1f8c4a0fdcb6498eb058d5, UNASSIGN 2024-11-25T03:38:20,209 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=c6d2702d0a1f8c4a0fdcb6498eb058d5, regionState=CLOSING, regionLocation=38a5d2077268,34881,1732505806588 2024-11-25T03:38:20,210 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c6d2702d0a1f8c4a0fdcb6498eb058d5, UNASSIGN because future has completed 2024-11-25T03:38:20,210 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T03:38:20,210 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588}] 2024-11-25T03:38:20,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-25T03:38:20,362 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(122): Close c6d2702d0a1f8c4a0fdcb6498eb058d5 2024-11-25T03:38:20,362 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T03:38:20,362 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1722): Closing c6d2702d0a1f8c4a0fdcb6498eb058d5, disabling compactions & flushes 2024-11-25T03:38:20,362 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:20,362 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:20,362 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. after waiting 0 ms 2024-11-25T03:38:20,362 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:20,362 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(2902): Flushing c6d2702d0a1f8c4a0fdcb6498eb058d5 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-25T03:38:20,366 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/e36eea3d577242e2b699b63440a6e6e4 is 50, key is test_row_0/A:col10/1732505897458/Put/seqid=0 2024-11-25T03:38:20,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742138_1314 (size=12301) 2024-11-25T03:38:20,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-25T03:38:20,770 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/e36eea3d577242e2b699b63440a6e6e4 2024-11-25T03:38:20,778 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/e818724e511e48779ca715f3137506ae is 50, key is test_row_0/B:col10/1732505897458/Put/seqid=0 2024-11-25T03:38:20,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742139_1315 (size=12301) 2024-11-25T03:38:20,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-25T03:38:21,181 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/e818724e511e48779ca715f3137506ae 2024-11-25T03:38:21,188 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/86d09f8b7bba4faa835f5d244f3a556e is 50, key is test_row_0/C:col10/1732505897458/Put/seqid=0 2024-11-25T03:38:21,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742140_1316 (size=12301) 2024-11-25T03:38:21,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-25T03:38:21,591 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/86d09f8b7bba4faa835f5d244f3a556e 2024-11-25T03:38:21,595 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/A/e36eea3d577242e2b699b63440a6e6e4 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/e36eea3d577242e2b699b63440a6e6e4 2024-11-25T03:38:21,598 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/e36eea3d577242e2b699b63440a6e6e4, entries=150, sequenceid=325, filesize=12.0 K 2024-11-25T03:38:21,599 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/B/e818724e511e48779ca715f3137506ae as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/e818724e511e48779ca715f3137506ae 2024-11-25T03:38:21,602 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/e818724e511e48779ca715f3137506ae, entries=150, sequenceid=325, filesize=12.0 K 2024-11-25T03:38:21,603 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/.tmp/C/86d09f8b7bba4faa835f5d244f3a556e as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/86d09f8b7bba4faa835f5d244f3a556e 2024-11-25T03:38:21,606 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/86d09f8b7bba4faa835f5d244f3a556e, entries=150, sequenceid=325, filesize=12.0 K 2024-11-25T03:38:21,607 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for c6d2702d0a1f8c4a0fdcb6498eb058d5 in 1245ms, sequenceid=325, compaction requested=true 2024-11-25T03:38:21,607 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/6b4bcc156fcf4c99912a3bbc86f9a131, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/67835b1a0c68429bb42a58066522e38e, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/6af3d274bb3d4413920c757af6598b2f, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/c378854276e842d9ab252ef759ca52cc, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/21e8530fc0a64bf189e7778a2f8fcdc4, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/96f8c6ab33d84e5f84d1d0f0b6eaff88, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/966d63dd5e4445a1a1389d6f89b05d45, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/fd41e03de24546fa9344b3a8ee505a51, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/9b4c03842fab420ca462efb8c1102ac8, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/e7f25cc9a0cb4586b43abd2479a1dc89, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/4952b9ff5c024c38a4088aa176ab1631, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/bc17a0bef87f484fb1eedf95a66e149b, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/5debf147c8b1435582d96267291c8aba, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/13461dee72a24f809b286de242003a4b, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/c73036f1bee64e7ebe91de68c327a073, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/68a65058743e4e6e8c2f10ab73f254ab, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/a264cb364d024e7d9b60aa06adc9ed63, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/53d4ac435d7d413d8abd396ae66e6055, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/d23bb190567344c59cafba7ec6c934c5, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/a5b457cf672a4c659b100937ed670163] to archive 2024-11-25T03:38:21,608 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T03:38:21,609 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/6b4bcc156fcf4c99912a3bbc86f9a131 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/6b4bcc156fcf4c99912a3bbc86f9a131 2024-11-25T03:38:21,610 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/67835b1a0c68429bb42a58066522e38e to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/67835b1a0c68429bb42a58066522e38e 2024-11-25T03:38:21,611 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/6af3d274bb3d4413920c757af6598b2f to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/6af3d274bb3d4413920c757af6598b2f 2024-11-25T03:38:21,612 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/c378854276e842d9ab252ef759ca52cc to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/c378854276e842d9ab252ef759ca52cc 2024-11-25T03:38:21,613 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/21e8530fc0a64bf189e7778a2f8fcdc4 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/21e8530fc0a64bf189e7778a2f8fcdc4 2024-11-25T03:38:21,614 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/96f8c6ab33d84e5f84d1d0f0b6eaff88 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/96f8c6ab33d84e5f84d1d0f0b6eaff88 2024-11-25T03:38:21,615 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/966d63dd5e4445a1a1389d6f89b05d45 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/966d63dd5e4445a1a1389d6f89b05d45 2024-11-25T03:38:21,616 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/fd41e03de24546fa9344b3a8ee505a51 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/fd41e03de24546fa9344b3a8ee505a51 2024-11-25T03:38:21,617 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/9b4c03842fab420ca462efb8c1102ac8 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/9b4c03842fab420ca462efb8c1102ac8 2024-11-25T03:38:21,618 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/e7f25cc9a0cb4586b43abd2479a1dc89 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/e7f25cc9a0cb4586b43abd2479a1dc89 2024-11-25T03:38:21,619 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/4952b9ff5c024c38a4088aa176ab1631 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/4952b9ff5c024c38a4088aa176ab1631 2024-11-25T03:38:21,619 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/bc17a0bef87f484fb1eedf95a66e149b to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/bc17a0bef87f484fb1eedf95a66e149b 2024-11-25T03:38:21,620 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/5debf147c8b1435582d96267291c8aba to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/5debf147c8b1435582d96267291c8aba 2024-11-25T03:38:21,621 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/13461dee72a24f809b286de242003a4b to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/13461dee72a24f809b286de242003a4b 2024-11-25T03:38:21,622 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/c73036f1bee64e7ebe91de68c327a073 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/c73036f1bee64e7ebe91de68c327a073 2024-11-25T03:38:21,623 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/68a65058743e4e6e8c2f10ab73f254ab to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/68a65058743e4e6e8c2f10ab73f254ab 2024-11-25T03:38:21,624 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/a264cb364d024e7d9b60aa06adc9ed63 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/a264cb364d024e7d9b60aa06adc9ed63 2024-11-25T03:38:21,625 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/53d4ac435d7d413d8abd396ae66e6055 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/53d4ac435d7d413d8abd396ae66e6055 2024-11-25T03:38:21,626 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/d23bb190567344c59cafba7ec6c934c5 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/d23bb190567344c59cafba7ec6c934c5 2024-11-25T03:38:21,627 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/a5b457cf672a4c659b100937ed670163 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/a5b457cf672a4c659b100937ed670163 2024-11-25T03:38:21,628 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/ff8674afbcb94968b039ec8b7a6323e9, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/75777a61a1a24c5d8ed5904e1c698d92, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/fde8222fef134fffbe4430bcd68f194d, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/673c8a12e77a4a63a637dc4eefd877fa, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/8d42727018454c02b1263e3f715e6fc7, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/504a74083c7048e5a577564e770d2306, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/3340461bd28d4460980a317a0844330a, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/ed347874d15d44c9bc5a873a8c32818b, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/56b512508579434f9764abb04eb7834c, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/5ac60418511549b2b7a5cccbbd7a6b2f, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/773d5fb95697427ab1d0dd17a204a92d, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/b56b9ccfd2b548779dae3e7b4137b812, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/02f8d012f3c6443198c37bf8cfe3cc77, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/3e64b9b076554ca59f76fcce14a6b4ec, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/470dc5ce4424409690bf20ff57599044, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/87d4adb896a945cabe4f620fb8ea11f7, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/2bf751a1bd3a42b190a5cc7ecac648b4, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/6002a427a0c14c41a57a9df0eca3123c, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/da6e41961c654033a4f81eda2d84c3b4, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/9416cf1990b74663881d2079c0ed2b96] to archive 2024-11-25T03:38:21,629 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T03:38:21,630 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/ff8674afbcb94968b039ec8b7a6323e9 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/ff8674afbcb94968b039ec8b7a6323e9 2024-11-25T03:38:21,631 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/75777a61a1a24c5d8ed5904e1c698d92 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/75777a61a1a24c5d8ed5904e1c698d92 2024-11-25T03:38:21,632 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/fde8222fef134fffbe4430bcd68f194d to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/fde8222fef134fffbe4430bcd68f194d 2024-11-25T03:38:21,632 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/673c8a12e77a4a63a637dc4eefd877fa to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/673c8a12e77a4a63a637dc4eefd877fa 2024-11-25T03:38:21,633 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/8d42727018454c02b1263e3f715e6fc7 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/8d42727018454c02b1263e3f715e6fc7 2024-11-25T03:38:21,634 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/504a74083c7048e5a577564e770d2306 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/504a74083c7048e5a577564e770d2306 2024-11-25T03:38:21,635 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/3340461bd28d4460980a317a0844330a to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/3340461bd28d4460980a317a0844330a 2024-11-25T03:38:21,636 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/ed347874d15d44c9bc5a873a8c32818b to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/ed347874d15d44c9bc5a873a8c32818b 2024-11-25T03:38:21,637 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/56b512508579434f9764abb04eb7834c to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/56b512508579434f9764abb04eb7834c 2024-11-25T03:38:21,638 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/5ac60418511549b2b7a5cccbbd7a6b2f to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/5ac60418511549b2b7a5cccbbd7a6b2f 2024-11-25T03:38:21,639 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/773d5fb95697427ab1d0dd17a204a92d to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/773d5fb95697427ab1d0dd17a204a92d 2024-11-25T03:38:21,640 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/b56b9ccfd2b548779dae3e7b4137b812 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/b56b9ccfd2b548779dae3e7b4137b812 2024-11-25T03:38:21,641 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/02f8d012f3c6443198c37bf8cfe3cc77 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/02f8d012f3c6443198c37bf8cfe3cc77 2024-11-25T03:38:21,643 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/3e64b9b076554ca59f76fcce14a6b4ec to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/3e64b9b076554ca59f76fcce14a6b4ec 2024-11-25T03:38:21,644 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/470dc5ce4424409690bf20ff57599044 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/470dc5ce4424409690bf20ff57599044 2024-11-25T03:38:21,645 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/87d4adb896a945cabe4f620fb8ea11f7 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/87d4adb896a945cabe4f620fb8ea11f7 2024-11-25T03:38:21,646 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/2bf751a1bd3a42b190a5cc7ecac648b4 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/2bf751a1bd3a42b190a5cc7ecac648b4 2024-11-25T03:38:21,647 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/6002a427a0c14c41a57a9df0eca3123c to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/6002a427a0c14c41a57a9df0eca3123c 2024-11-25T03:38:21,648 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/da6e41961c654033a4f81eda2d84c3b4 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/da6e41961c654033a4f81eda2d84c3b4 2024-11-25T03:38:21,650 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/9416cf1990b74663881d2079c0ed2b96 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/9416cf1990b74663881d2079c0ed2b96 2024-11-25T03:38:21,651 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/67a1b3058b724c33bc2e579763a2258b, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/93850fe96194428a8778fa43245765bd, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/77468b27b66848bfa4d48b18994813c2, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/506b368e16df4068aa92fd523f2199e3, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/934d0c6c6f274a73adab01b5f52924cd, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/860f0710163247089acb8ad7122ecb8e, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/d584c43b2c364b59a521d3caeb0c8b61, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/70d63b66c8f041eb839791c92954a3fb, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/b2b608f83bd84a4d9015addd9e6c5981, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/c50ea88784d94720b08b8d3b6bec91a1, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/46a9df7a45c140bd8dc5cceb90d1543c, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/29e2971d4ae443cc83491d0dcb62f894, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/21177cac61484ff5ba10734aadb3cfda, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/ea9957fd23514cbfa0c3fdade92d487b, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/a08a68bbfa8e461f9beae6916d8fbf79, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/984153367ac5446a888b9c138fcb7f70, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/c4e8bee812a6485f9dc974c5c3bed1ef, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/2f968a32b6ab4f7daf7de1cd37d585d5, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/8964a75802174316ba7850736d416d9b, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/aadccdfb5554497492d50f1d5012c36b] to archive 2024-11-25T03:38:21,652 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T03:38:21,653 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/67a1b3058b724c33bc2e579763a2258b to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/67a1b3058b724c33bc2e579763a2258b 2024-11-25T03:38:21,654 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/93850fe96194428a8778fa43245765bd to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/93850fe96194428a8778fa43245765bd 2024-11-25T03:38:21,656 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/77468b27b66848bfa4d48b18994813c2 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/77468b27b66848bfa4d48b18994813c2 2024-11-25T03:38:21,657 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/506b368e16df4068aa92fd523f2199e3 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/506b368e16df4068aa92fd523f2199e3 2024-11-25T03:38:21,658 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/934d0c6c6f274a73adab01b5f52924cd to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/934d0c6c6f274a73adab01b5f52924cd 2024-11-25T03:38:21,659 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/860f0710163247089acb8ad7122ecb8e to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/860f0710163247089acb8ad7122ecb8e 2024-11-25T03:38:21,660 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/d584c43b2c364b59a521d3caeb0c8b61 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/d584c43b2c364b59a521d3caeb0c8b61 2024-11-25T03:38:21,661 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/70d63b66c8f041eb839791c92954a3fb to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/70d63b66c8f041eb839791c92954a3fb 2024-11-25T03:38:21,662 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/b2b608f83bd84a4d9015addd9e6c5981 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/b2b608f83bd84a4d9015addd9e6c5981 2024-11-25T03:38:21,663 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/c50ea88784d94720b08b8d3b6bec91a1 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/c50ea88784d94720b08b8d3b6bec91a1 2024-11-25T03:38:21,664 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/46a9df7a45c140bd8dc5cceb90d1543c to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/46a9df7a45c140bd8dc5cceb90d1543c 2024-11-25T03:38:21,665 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/29e2971d4ae443cc83491d0dcb62f894 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/29e2971d4ae443cc83491d0dcb62f894 2024-11-25T03:38:21,666 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/21177cac61484ff5ba10734aadb3cfda to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/21177cac61484ff5ba10734aadb3cfda 2024-11-25T03:38:21,666 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/ea9957fd23514cbfa0c3fdade92d487b to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/ea9957fd23514cbfa0c3fdade92d487b 2024-11-25T03:38:21,667 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/a08a68bbfa8e461f9beae6916d8fbf79 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/a08a68bbfa8e461f9beae6916d8fbf79 2024-11-25T03:38:21,668 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/984153367ac5446a888b9c138fcb7f70 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/984153367ac5446a888b9c138fcb7f70 2024-11-25T03:38:21,669 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/c4e8bee812a6485f9dc974c5c3bed1ef to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/c4e8bee812a6485f9dc974c5c3bed1ef 2024-11-25T03:38:21,670 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/2f968a32b6ab4f7daf7de1cd37d585d5 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/2f968a32b6ab4f7daf7de1cd37d585d5 2024-11-25T03:38:21,671 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/8964a75802174316ba7850736d416d9b to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/8964a75802174316ba7850736d416d9b 2024-11-25T03:38:21,672 DEBUG [StoreCloser-TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/aadccdfb5554497492d50f1d5012c36b to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/aadccdfb5554497492d50f1d5012c36b 2024-11-25T03:38:21,677 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/recovered.edits/328.seqid, newMaxSeqId=328, maxSeqId=1 2024-11-25T03:38:21,677 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. 2024-11-25T03:38:21,677 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1676): Region close journal for c6d2702d0a1f8c4a0fdcb6498eb058d5: Waiting for close lock at 1732505900362Running coprocessor pre-close hooks at 1732505900362Disabling compacts and flushes for region at 1732505900362Disabling writes for close at 1732505900362Obtaining lock to block concurrent updates at 1732505900362Preparing flush snapshotting stores in c6d2702d0a1f8c4a0fdcb6498eb058d5 at 1732505900362Finished memstore snapshotting TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732505900363 (+1 ms)Flushing stores of TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5. at 1732505900363Flushing c6d2702d0a1f8c4a0fdcb6498eb058d5/A: creating writer at 1732505900363Flushing c6d2702d0a1f8c4a0fdcb6498eb058d5/A: appending metadata at 1732505900366 (+3 ms)Flushing c6d2702d0a1f8c4a0fdcb6498eb058d5/A: closing flushed file at 1732505900366Flushing c6d2702d0a1f8c4a0fdcb6498eb058d5/B: creating writer at 1732505900774 (+408 ms)Flushing c6d2702d0a1f8c4a0fdcb6498eb058d5/B: appending metadata at 1732505900777 (+3 ms)Flushing c6d2702d0a1f8c4a0fdcb6498eb058d5/B: closing flushed file at 1732505900777Flushing c6d2702d0a1f8c4a0fdcb6498eb058d5/C: creating writer at 1732505901185 (+408 ms)Flushing c6d2702d0a1f8c4a0fdcb6498eb058d5/C: appending metadata at 1732505901187 (+2 ms)Flushing c6d2702d0a1f8c4a0fdcb6498eb058d5/C: closing flushed file at 1732505901187Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1cc93ec2: reopening flushed file at 1732505901595 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7aa3b7b6: reopening flushed file at 1732505901599 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15d59ffb: reopening flushed file at 1732505901602 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for c6d2702d0a1f8c4a0fdcb6498eb058d5 in 1245ms, sequenceid=325, compaction requested=true at 1732505901607 (+5 ms)Writing region close event to WAL at 1732505901673 (+66 ms)Running coprocessor post-close hooks at 1732505901677 (+4 ms)Closed at 1732505901677 2024-11-25T03:38:21,680 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(157): Closed c6d2702d0a1f8c4a0fdcb6498eb058d5 2024-11-25T03:38:21,680 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=c6d2702d0a1f8c4a0fdcb6498eb058d5, regionState=CLOSED 2024-11-25T03:38:21,682 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 because future has completed 2024-11-25T03:38:21,684 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-25T03:38:21,684 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseRegionProcedure c6d2702d0a1f8c4a0fdcb6498eb058d5, server=38a5d2077268,34881,1732505806588 in 1.4730 sec 2024-11-25T03:38:21,686 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-11-25T03:38:21,686 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c6d2702d0a1f8c4a0fdcb6498eb058d5, UNASSIGN in 1.4770 sec 2024-11-25T03:38:21,688 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-25T03:38:21,688 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4800 sec 2024-11-25T03:38:21,689 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732505901689"}]},"ts":"1732505901689"} 2024-11-25T03:38:21,690 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-25T03:38:21,690 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-25T03:38:21,691 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4890 sec 2024-11-25T03:38:22,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-25T03:38:22,348 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:38:22,348 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-25T03:38:22,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:38:22,350 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:38:22,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-25T03:38:22,351 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=84, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:38:22,354 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5 2024-11-25T03:38:22,356 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A, FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B, FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C, FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/recovered.edits] 2024-11-25T03:38:22,360 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/572f4686fb9e4ad28990301956e9f6fc to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/572f4686fb9e4ad28990301956e9f6fc 2024-11-25T03:38:22,361 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/9c2159d85ea441a4850869b766159226 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/9c2159d85ea441a4850869b766159226 2024-11-25T03:38:22,362 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/e36eea3d577242e2b699b63440a6e6e4 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/A/e36eea3d577242e2b699b63440a6e6e4 2024-11-25T03:38:22,365 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/2dd8dd54e9cb45829a5f566b0a15909e to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/2dd8dd54e9cb45829a5f566b0a15909e 2024-11-25T03:38:22,366 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/d590cbcb85064a4bb6cab27979aacf37 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/d590cbcb85064a4bb6cab27979aacf37 2024-11-25T03:38:22,368 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/e818724e511e48779ca715f3137506ae to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/B/e818724e511e48779ca715f3137506ae 2024-11-25T03:38:22,370 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/12d38c447ad94e9884a0f43870a20ba4 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/12d38c447ad94e9884a0f43870a20ba4 2024-11-25T03:38:22,372 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/86d09f8b7bba4faa835f5d244f3a556e to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/86d09f8b7bba4faa835f5d244f3a556e 2024-11-25T03:38:22,373 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/af240d7aaa654b679e66c499b5a6eee0 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/C/af240d7aaa654b679e66c499b5a6eee0 2024-11-25T03:38:22,377 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/recovered.edits/328.seqid to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5/recovered.edits/328.seqid 2024-11-25T03:38:22,377 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/c6d2702d0a1f8c4a0fdcb6498eb058d5 2024-11-25T03:38:22,378 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-25T03:38:22,380 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=84, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:38:22,383 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-25T03:38:22,386 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-25T03:38:22,388 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=84, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:38:22,388 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-25T03:38:22,388 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732505902388"}]},"ts":"9223372036854775807"} 2024-11-25T03:38:22,390 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-25T03:38:22,390 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c6d2702d0a1f8c4a0fdcb6498eb058d5, NAME => 'TestAcidGuarantees,,1732505873652.c6d2702d0a1f8c4a0fdcb6498eb058d5.', STARTKEY => '', ENDKEY => ''}] 2024-11-25T03:38:22,390 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-25T03:38:22,390 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732505902390"}]},"ts":"9223372036854775807"} 2024-11-25T03:38:22,392 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-25T03:38:22,393 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=84, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:38:22,394 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 45 msec 2024-11-25T03:38:22,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-25T03:38:22,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-25T03:38:22,458 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:38:22,469 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=238 (was 238), OpenFileDescriptor=453 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1943 (was 935) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7183 (was 5960) - AvailableMemoryMB LEAK? - 2024-11-25T03:38:22,480 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=238, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=1943, ProcessCount=11, AvailableMemoryMB=7182 2024-11-25T03:38:22,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T03:38:22,482 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T03:38:22,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-25T03:38:22,484 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T03:38:22,484 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:22,484 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 85 2024-11-25T03:38:22,485 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T03:38:22,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-25T03:38:22,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742141_1317 (size=959) 2024-11-25T03:38:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-25T03:38:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-25T03:38:22,893 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3 2024-11-25T03:38:22,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742142_1318 (size=53) 2024-11-25T03:38:23,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-25T03:38:23,302 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:38:23,302 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 629be9b3b266d56b676cd5a9595f19cd, disabling compactions & flushes 2024-11-25T03:38:23,302 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:23,302 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:23,302 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. after waiting 0 ms 2024-11-25T03:38:23,302 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:23,302 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:23,302 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 629be9b3b266d56b676cd5a9595f19cd: Waiting for close lock at 1732505903302Disabling compacts and flushes for region at 1732505903302Disabling writes for close at 1732505903302Writing region close event to WAL at 1732505903302Closed at 1732505903302 2024-11-25T03:38:23,303 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T03:38:23,304 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732505903303"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732505903303"}]},"ts":"1732505903303"} 2024-11-25T03:38:23,305 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T03:38:23,306 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T03:38:23,306 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732505903306"}]},"ts":"1732505903306"} 2024-11-25T03:38:23,307 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-25T03:38:23,308 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=629be9b3b266d56b676cd5a9595f19cd, ASSIGN}] 2024-11-25T03:38:23,309 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=629be9b3b266d56b676cd5a9595f19cd, ASSIGN 2024-11-25T03:38:23,310 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=629be9b3b266d56b676cd5a9595f19cd, ASSIGN; state=OFFLINE, location=38a5d2077268,34881,1732505806588; forceNewPlan=false, retain=false 2024-11-25T03:38:23,460 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=629be9b3b266d56b676cd5a9595f19cd, regionState=OPENING, regionLocation=38a5d2077268,34881,1732505806588 2024-11-25T03:38:23,462 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=629be9b3b266d56b676cd5a9595f19cd, ASSIGN because future has completed 2024-11-25T03:38:23,462 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588}] 2024-11-25T03:38:23,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-25T03:38:23,618 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:23,618 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7752): Opening region: {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:38:23,618 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:23,618 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:38:23,618 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7794): checking encryption for 629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:23,618 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7797): checking classloading for 629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:23,619 INFO [StoreOpener-629be9b3b266d56b676cd5a9595f19cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:23,621 INFO [StoreOpener-629be9b3b266d56b676cd5a9595f19cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 629be9b3b266d56b676cd5a9595f19cd columnFamilyName A 2024-11-25T03:38:23,621 DEBUG [StoreOpener-629be9b3b266d56b676cd5a9595f19cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:23,621 INFO [StoreOpener-629be9b3b266d56b676cd5a9595f19cd-1 {}] regionserver.HStore(327): Store=629be9b3b266d56b676cd5a9595f19cd/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:38:23,621 INFO [StoreOpener-629be9b3b266d56b676cd5a9595f19cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:23,622 INFO [StoreOpener-629be9b3b266d56b676cd5a9595f19cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 629be9b3b266d56b676cd5a9595f19cd columnFamilyName B 2024-11-25T03:38:23,622 DEBUG [StoreOpener-629be9b3b266d56b676cd5a9595f19cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:23,623 INFO [StoreOpener-629be9b3b266d56b676cd5a9595f19cd-1 {}] regionserver.HStore(327): Store=629be9b3b266d56b676cd5a9595f19cd/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:38:23,623 INFO [StoreOpener-629be9b3b266d56b676cd5a9595f19cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:23,624 INFO [StoreOpener-629be9b3b266d56b676cd5a9595f19cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 629be9b3b266d56b676cd5a9595f19cd columnFamilyName C 2024-11-25T03:38:23,624 DEBUG [StoreOpener-629be9b3b266d56b676cd5a9595f19cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:23,624 INFO [StoreOpener-629be9b3b266d56b676cd5a9595f19cd-1 {}] regionserver.HStore(327): Store=629be9b3b266d56b676cd5a9595f19cd/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:38:23,624 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1038): replaying wal for 629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:23,625 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:23,625 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:23,626 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1048): stopping wal replay for 629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:23,626 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1060): Cleaning up temporary data for 629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:23,626 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T03:38:23,627 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1093): writing seq id for 629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:23,629 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:38:23,629 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1114): Opened 629be9b3b266d56b676cd5a9595f19cd; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71175392, jitterRate=0.06059598922729492}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T03:38:23,629 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:23,630 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1006): Region open journal for 629be9b3b266d56b676cd5a9595f19cd: Running coprocessor pre-open hook at 1732505903618Writing region info on filesystem at 1732505903618Initializing all the Stores at 1732505903619 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732505903619Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732505903619Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732505903619Cleaning up temporary data from old regions at 1732505903626 (+7 ms)Running coprocessor post-open hooks at 1732505903629 (+3 ms)Region opened successfully at 1732505903630 (+1 ms) 2024-11-25T03:38:23,631 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., pid=87, masterSystemTime=1732505903614 2024-11-25T03:38:23,632 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:23,632 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:23,633 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=629be9b3b266d56b676cd5a9595f19cd, regionState=OPEN, openSeqNum=2, regionLocation=38a5d2077268,34881,1732505806588 2024-11-25T03:38:23,634 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 because future has completed 2024-11-25T03:38:23,636 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-25T03:38:23,636 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; OpenRegionProcedure 629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 in 173 msec 2024-11-25T03:38:23,638 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-25T03:38:23,638 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=629be9b3b266d56b676cd5a9595f19cd, ASSIGN in 329 msec 2024-11-25T03:38:23,639 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T03:38:23,639 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732505903639"}]},"ts":"1732505903639"} 2024-11-25T03:38:23,640 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-25T03:38:23,641 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T03:38:23,642 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1590 sec 2024-11-25T03:38:24,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-25T03:38:24,638 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:38:24,639 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@283b5885, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:24,640 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:38:24,640 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:38:24,641 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:38:24,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:38:24,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:38:24,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@784592c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:24,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:38:24,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:38:24,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:24,643 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55318, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:38:24,644 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@282c63f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:24,644 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:38:24,645 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:38:24,645 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:38:24,646 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36680, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:38:24,647 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T03:38:24,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 38a5d2077268,42243,1732505805780 2024-11-25T03:38:24,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5539dfd 2024-11-25T03:38:24,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T03:38:24,648 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55332, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T03:38:24,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T03:38:24,650 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T03:38:24,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-25T03:38:24,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742143_1319 (size=995) 2024-11-25T03:38:25,068 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-25T03:38:25,068 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-25T03:38:25,070 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T03:38:25,073 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=629be9b3b266d56b676cd5a9595f19cd, REOPEN/MOVE}] 2024-11-25T03:38:25,074 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=629be9b3b266d56b676cd5a9595f19cd, REOPEN/MOVE 2024-11-25T03:38:25,075 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=629be9b3b266d56b676cd5a9595f19cd, regionState=CLOSING, regionLocation=38a5d2077268,34881,1732505806588 2024-11-25T03:38:25,076 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=629be9b3b266d56b676cd5a9595f19cd, REOPEN/MOVE because future has completed 2024-11-25T03:38:25,077 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T03:38:25,077 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588}] 2024-11-25T03:38:25,229 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(122): Close 629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:25,229 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T03:38:25,229 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1722): Closing 629be9b3b266d56b676cd5a9595f19cd, disabling compactions & flushes 2024-11-25T03:38:25,229 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:25,229 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:25,229 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. after waiting 0 ms 2024-11-25T03:38:25,229 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:25,233 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-25T03:38:25,234 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:25,234 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1676): Region close journal for 629be9b3b266d56b676cd5a9595f19cd: Waiting for close lock at 1732505905229Running coprocessor pre-close hooks at 1732505905229Disabling compacts and flushes for region at 1732505905229Disabling writes for close at 1732505905229Writing region close event to WAL at 1732505905230 (+1 ms)Running coprocessor post-close hooks at 1732505905234 (+4 ms)Closed at 1732505905234 2024-11-25T03:38:25,234 WARN [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionServer(3299): Not adding moved region record: 629be9b3b266d56b676cd5a9595f19cd to self. 2024-11-25T03:38:25,236 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(157): Closed 629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:25,237 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=629be9b3b266d56b676cd5a9595f19cd, regionState=CLOSED 2024-11-25T03:38:25,238 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 because future has completed 2024-11-25T03:38:25,241 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-25T03:38:25,241 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseRegionProcedure 629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 in 162 msec 2024-11-25T03:38:25,242 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=629be9b3b266d56b676cd5a9595f19cd, REOPEN/MOVE; state=CLOSED, location=38a5d2077268,34881,1732505806588; forceNewPlan=false, retain=true 2024-11-25T03:38:25,393 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=629be9b3b266d56b676cd5a9595f19cd, regionState=OPENING, regionLocation=38a5d2077268,34881,1732505806588 2024-11-25T03:38:25,395 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=629be9b3b266d56b676cd5a9595f19cd, REOPEN/MOVE because future has completed 2024-11-25T03:38:25,395 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588}] 2024-11-25T03:38:25,550 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:25,551 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7752): Opening region: {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:38:25,551 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:25,551 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:38:25,551 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7794): checking encryption for 629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:25,551 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7797): checking classloading for 629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:25,553 INFO [StoreOpener-629be9b3b266d56b676cd5a9595f19cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:25,553 INFO [StoreOpener-629be9b3b266d56b676cd5a9595f19cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 629be9b3b266d56b676cd5a9595f19cd columnFamilyName A 2024-11-25T03:38:25,554 DEBUG [StoreOpener-629be9b3b266d56b676cd5a9595f19cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:25,555 INFO [StoreOpener-629be9b3b266d56b676cd5a9595f19cd-1 {}] regionserver.HStore(327): Store=629be9b3b266d56b676cd5a9595f19cd/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:38:25,556 INFO [StoreOpener-629be9b3b266d56b676cd5a9595f19cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:25,557 INFO [StoreOpener-629be9b3b266d56b676cd5a9595f19cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 629be9b3b266d56b676cd5a9595f19cd columnFamilyName B 2024-11-25T03:38:25,557 DEBUG [StoreOpener-629be9b3b266d56b676cd5a9595f19cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:25,557 INFO [StoreOpener-629be9b3b266d56b676cd5a9595f19cd-1 {}] regionserver.HStore(327): Store=629be9b3b266d56b676cd5a9595f19cd/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:38:25,557 INFO [StoreOpener-629be9b3b266d56b676cd5a9595f19cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:25,558 INFO [StoreOpener-629be9b3b266d56b676cd5a9595f19cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 629be9b3b266d56b676cd5a9595f19cd columnFamilyName C 2024-11-25T03:38:25,558 DEBUG [StoreOpener-629be9b3b266d56b676cd5a9595f19cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:25,558 INFO [StoreOpener-629be9b3b266d56b676cd5a9595f19cd-1 {}] regionserver.HStore(327): Store=629be9b3b266d56b676cd5a9595f19cd/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:38:25,558 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1038): replaying wal for 629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:25,559 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:25,559 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:25,560 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1048): stopping wal replay for 629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:25,560 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1060): Cleaning up temporary data for 629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:25,561 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T03:38:25,562 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1093): writing seq id for 629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:25,562 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1114): Opened 629be9b3b266d56b676cd5a9595f19cd; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59416872, jitterRate=-0.11461961269378662}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T03:38:25,562 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:25,563 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1006): Region open journal for 629be9b3b266d56b676cd5a9595f19cd: Running coprocessor pre-open hook at 1732505905551Writing region info on filesystem at 1732505905551Initializing all the Stores at 1732505905552 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732505905552Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732505905552Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732505905553 (+1 ms)Cleaning up temporary data from old regions at 1732505905560 (+7 ms)Running coprocessor post-open hooks at 1732505905562 (+2 ms)Region opened successfully at 1732505905563 (+1 ms) 2024-11-25T03:38:25,564 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., pid=92, masterSystemTime=1732505905547 2024-11-25T03:38:25,566 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:25,566 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:25,566 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=629be9b3b266d56b676cd5a9595f19cd, regionState=OPEN, openSeqNum=5, regionLocation=38a5d2077268,34881,1732505806588 2024-11-25T03:38:25,567 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 because future has completed 2024-11-25T03:38:25,569 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=90 2024-11-25T03:38:25,569 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure 629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 in 173 msec 2024-11-25T03:38:25,571 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-25T03:38:25,571 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=629be9b3b266d56b676cd5a9595f19cd, REOPEN/MOVE in 496 msec 2024-11-25T03:38:25,573 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-25T03:38:25,573 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 501 msec 2024-11-25T03:38:25,574 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 922 msec 2024-11-25T03:38:25,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-25T03:38:25,575 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:38:25,576 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cf7f63f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:25,576 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:38:25,576 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:38:25,577 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:38:25,577 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:38:25,577 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:38:25,577 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e15df87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:25,577 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:38:25,577 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:38:25,578 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:25,578 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55358, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:38:25,579 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@427de420, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:25,579 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4600746, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:25,579 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:38:25,579 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:38:25,580 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:38:25,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:38:25,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:38:25,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1302d318, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:25,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:38:25,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:38:25,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:25,581 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55374, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:38:25,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a40f141, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:25,582 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6447def3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:25,582 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:38:25,582 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:38:25,583 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:38:25,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:38:25,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:38:25,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47144b4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:25,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:38:25,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:38:25,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:25,584 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55394, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:38:25,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a048ff1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:25,585 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5345722e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:25,586 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:38:25,586 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:38:25,586 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:38:25,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:38:25,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:38:25,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@360d7647, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:25,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:38:25,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:38:25,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:25,587 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55408, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:38:25,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e18adc5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:25,588 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dd3a90f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:25,588 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:38:25,588 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:38:25,589 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:38:25,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:38:25,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:38:25,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@366e06ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:25,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:38:25,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:38:25,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:25,590 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55428, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:38:25,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72271b8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:25,591 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eba3e2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:25,591 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:38:25,591 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:38:25,592 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:38:25,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:38:25,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:38:25,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@696ad0bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:25,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:38:25,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:38:25,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:25,593 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55444, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:38:25,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@588e5314, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:25,594 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79570776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:25,594 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:38:25,594 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:38:25,595 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:38:25,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:38:25,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:38:25,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@424346e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:25,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:38:25,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:38:25,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:25,596 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55452, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:38:25,596 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5424912b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:25,597 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43065a6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:25,597 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:38:25,597 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:38:25,597 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:38:25,597 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:38:25,597 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:38:25,597 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43068765, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:25,598 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:38:25,598 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:38:25,598 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:25,598 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55462, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:38:25,599 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62f05c25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:25,599 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24bcd92c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:25,599 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:38:25,599 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:38:25,600 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:38:25,600 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:38:25,600 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:38:25,600 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1977f314, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:25,600 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:38:25,600 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:38:25,601 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:25,601 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55470, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:38:25,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f6c66a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:25,604 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b779047, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:25,604 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:38:25,604 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:38:25,604 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:38:25,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:38:25,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:38:25,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3daa098b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:25,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:38:25,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:38:25,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:25,606 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55488, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:38:25,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a651a15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:25,608 DEBUG [Thread-1474 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:38:25,608 DEBUG [Thread-1485 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:38:25,608 DEBUG [Thread-1487 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:38:25,608 DEBUG [Thread-1476 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:38:25,608 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:38:25,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:38:25,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:38:25,609 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:38:25,609 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:38:25,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-25T03:38:25,610 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:38:25,610 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:38:25,611 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36700, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:38:25,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-25T03:38:25,611 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:38:25,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:38:25,611 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:38:25,612 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:38:25,616 DEBUG [Thread-1479 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:38:25,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5] 2024-11-25T03:38:25,616 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5] 2024-11-25T03:38:25,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:38:25,617 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36708, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:38:25,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:38:25,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:38:25,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:38:25,619 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36712, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:38:25,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5] 2024-11-25T03:38:25,620 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36728, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:38:25,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5] 2024-11-25T03:38:25,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5] 2024-11-25T03:38:25,627 DEBUG [Thread-1483 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:38:25,628 DEBUG [Thread-1481 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:38:25,628 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:38:25,628 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:38:25,629 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:38:25,629 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:38:25,630 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36738, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:38:25,631 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36752, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:38:25,631 DEBUG [Thread-1472 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:38:25,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5] 2024-11-25T03:38:25,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:38:25,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5] 2024-11-25T03:38:25,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:38:25,634 DEBUG [Thread-1468 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:38:25,634 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36760, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:38:25,635 DEBUG [Thread-1470 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:38:25,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:38:25,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:38:25,636 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:38:25,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5] 2024-11-25T03:38:25,636 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:38:25,639 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36788, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:38:25,639 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36772, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:38:25,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5] 2024-11-25T03:38:25,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5] 2024-11-25T03:38:25,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:25,668 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:38:25,668 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 629be9b3b266d56b676cd5a9595f19cd 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-25T03:38:25,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411257a1fd224af0248f59a35c795dd2a42fd_629be9b3b266d56b676cd5a9595f19cd is 50, key is test_row_0/A:col10/1732505905666/Put/seqid=0 2024-11-25T03:38:25,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742144_1320 (size=14594) 2024-11-25T03:38:25,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-25T03:38:25,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-25T03:38:25,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:25,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. as already flushing 2024-11-25T03:38:25,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:25,764 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:25,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:25,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:25,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:25,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505965786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:25,798 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:25,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:25,798 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:25,798 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:25,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505965787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:25,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:25,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:25,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:25,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:25,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505965784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:25,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:25,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:25,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:25,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:25,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36696 deadline: 1732505965798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:25,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:25,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:25,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:25,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:25,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36760 deadline: 1732505965801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:25,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:25,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:25,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:25,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:25,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36696 deadline: 1732505965909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:25,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:25,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:25,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:25,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505965909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:25,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:25,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:25,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505965909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:25,912 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:25,912 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:25,912 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:25,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:25,913 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:25,913 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:25,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505965909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:25,913 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:25,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:25,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:25,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:25,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-25T03:38:25,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:25,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. as already flushing 2024-11-25T03:38:25,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:25,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:25,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:25,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:25,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-25T03:38:25,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:25,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36760 deadline: 1732505965927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:25,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:25,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:25,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:26,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-25T03:38:26,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:26,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. as already flushing 2024-11-25T03:38:26,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:26,076 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:26,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:26,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:26,107 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:26,112 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411257a1fd224af0248f59a35c795dd2a42fd_629be9b3b266d56b676cd5a9595f19cd to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411257a1fd224af0248f59a35c795dd2a42fd_629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:26,114 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/12fecffaebcd49cc8dc9d9f003bc5f70, store: [table=TestAcidGuarantees family=A region=629be9b3b266d56b676cd5a9595f19cd] 2024-11-25T03:38:26,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/12fecffaebcd49cc8dc9d9f003bc5f70 is 175, key is test_row_0/A:col10/1732505905666/Put/seqid=0 2024-11-25T03:38:26,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:26,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36696 deadline: 1732505966117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:26,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:26,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505966117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:26,126 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:26,126 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:26,126 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:26,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:26,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505966118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:26,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:26,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:26,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:26,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:26,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505966118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:26,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:26,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:26,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:26,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:26,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:26,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:26,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742145_1321 (size=39549) 2024-11-25T03:38:26,152 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/12fecffaebcd49cc8dc9d9f003bc5f70 2024-11-25T03:38:26,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:26,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36760 deadline: 1732505966147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:26,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:26,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:26,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:26,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/b474b691a1cb4e8883d8511a7ecdd6ca is 50, key is test_row_0/B:col10/1732505905666/Put/seqid=0 2024-11-25T03:38:26,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742146_1322 (size=12001) 2024-11-25T03:38:26,199 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/b474b691a1cb4e8883d8511a7ecdd6ca 2024-11-25T03:38:26,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-25T03:38:26,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:26,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. as already flushing 2024-11-25T03:38:26,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:26,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:26,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:26,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-25T03:38:26,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/c3e4165eb99643639fdb5496dceee90b is 50, key is test_row_0/C:col10/1732505905666/Put/seqid=0 2024-11-25T03:38:26,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742147_1323 (size=12001) 2024-11-25T03:38:26,260 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/c3e4165eb99643639fdb5496dceee90b 2024-11-25T03:38:26,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/12fecffaebcd49cc8dc9d9f003bc5f70 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/12fecffaebcd49cc8dc9d9f003bc5f70 2024-11-25T03:38:26,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/12fecffaebcd49cc8dc9d9f003bc5f70, entries=200, sequenceid=15, filesize=38.6 K 2024-11-25T03:38:26,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/b474b691a1cb4e8883d8511a7ecdd6ca as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/b474b691a1cb4e8883d8511a7ecdd6ca 2024-11-25T03:38:26,279 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/b474b691a1cb4e8883d8511a7ecdd6ca, entries=150, sequenceid=15, filesize=11.7 K 2024-11-25T03:38:26,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/c3e4165eb99643639fdb5496dceee90b as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/c3e4165eb99643639fdb5496dceee90b 2024-11-25T03:38:26,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/c3e4165eb99643639fdb5496dceee90b, entries=150, sequenceid=15, filesize=11.7 K 2024-11-25T03:38:26,285 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 629be9b3b266d56b676cd5a9595f19cd in 617ms, sequenceid=15, compaction requested=false 2024-11-25T03:38:26,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 629be9b3b266d56b676cd5a9595f19cd: 2024-11-25T03:38:26,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-25T03:38:26,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:26,382 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 629be9b3b266d56b676cd5a9595f19cd 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-25T03:38:26,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125b37cfde2adcc428dbabd8dad0b85ef73_629be9b3b266d56b676cd5a9595f19cd is 50, key is test_row_0/A:col10/1732505905797/Put/seqid=0 2024-11-25T03:38:26,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742148_1324 (size=12154) 2024-11-25T03:38:26,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:26,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. as already flushing 2024-11-25T03:38:26,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:26,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505966452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:26,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:26,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:26,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:26,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:26,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505966453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:26,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:26,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:26,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505966455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:26,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:26,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:26,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:26,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:26,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:26,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:26,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36696 deadline: 1732505966457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:26,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:26,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:26,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:26,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:26,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36760 deadline: 1732505966458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:26,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:26,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:26,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:26,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:26,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505966567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:26,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:26,574 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:26,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505966568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:26,574 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:26,574 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:26,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:26,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:26,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505966568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:26,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:26,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:26,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:26,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:26,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:26,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:26,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36696 deadline: 1732505966577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:26,586 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:26,586 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:26,586 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:26,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-25T03:38:26,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:26,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:26,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505966778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:26,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505966778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:26,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:26,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:26,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:26,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505966778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:26,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:26,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:26,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:26,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:26,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:26,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:26,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:26,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:26,799 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125b37cfde2adcc428dbabd8dad0b85ef73_629be9b3b266d56b676cd5a9595f19cd to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125b37cfde2adcc428dbabd8dad0b85ef73_629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:26,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/3772b5eaa7944a7394b50d74cf51964d, store: [table=TestAcidGuarantees family=A region=629be9b3b266d56b676cd5a9595f19cd] 2024-11-25T03:38:26,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/3772b5eaa7944a7394b50d74cf51964d is 175, key is test_row_0/A:col10/1732505905797/Put/seqid=0 2024-11-25T03:38:26,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:26,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36696 deadline: 1732505966797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:26,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:26,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:26,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:26,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742149_1325 (size=30955) 2024-11-25T03:38:26,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:26,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36760 deadline: 1732505966977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:26,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:26,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:26,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:27,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:27,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505967098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:27,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505967098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:27,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505967098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:27,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:27,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:27,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:27,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:27,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:27,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:27,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:27,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:27,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:27,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:27,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36696 deadline: 1732505967107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:27,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:27,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:27,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:27,199 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T03:38:27,206 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/3772b5eaa7944a7394b50d74cf51964d 2024-11-25T03:38:27,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/5bddac04ab7f448f85f14f3dd352b6e5 is 50, key is test_row_0/B:col10/1732505905797/Put/seqid=0 2024-11-25T03:38:27,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742150_1326 (size=12001) 2024-11-25T03:38:27,232 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/5bddac04ab7f448f85f14f3dd352b6e5 2024-11-25T03:38:27,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/9ff74abb6f5d4f67ad7f793b0a27e727 is 50, key is test_row_0/C:col10/1732505905797/Put/seqid=0 2024-11-25T03:38:27,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742151_1327 (size=12001) 2024-11-25T03:38:27,260 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/9ff74abb6f5d4f67ad7f793b0a27e727 2024-11-25T03:38:27,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/3772b5eaa7944a7394b50d74cf51964d as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/3772b5eaa7944a7394b50d74cf51964d 2024-11-25T03:38:27,270 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/3772b5eaa7944a7394b50d74cf51964d, entries=150, sequenceid=41, filesize=30.2 K 2024-11-25T03:38:27,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/5bddac04ab7f448f85f14f3dd352b6e5 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/5bddac04ab7f448f85f14f3dd352b6e5 2024-11-25T03:38:27,276 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/5bddac04ab7f448f85f14f3dd352b6e5, entries=150, sequenceid=41, filesize=11.7 K 2024-11-25T03:38:27,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/9ff74abb6f5d4f67ad7f793b0a27e727 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/9ff74abb6f5d4f67ad7f793b0a27e727 2024-11-25T03:38:27,283 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/9ff74abb6f5d4f67ad7f793b0a27e727, entries=150, sequenceid=41, filesize=11.7 K 2024-11-25T03:38:27,283 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 629be9b3b266d56b676cd5a9595f19cd in 901ms, sequenceid=41, compaction requested=false 2024-11-25T03:38:27,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 629be9b3b266d56b676cd5a9595f19cd: 2024-11-25T03:38:27,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:27,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-25T03:38:27,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-25T03:38:27,287 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-25T03:38:27,287 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6740 sec 2024-11-25T03:38:27,292 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 1.6820 sec 2024-11-25T03:38:27,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:27,611 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:38:27,612 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 629be9b3b266d56b676cd5a9595f19cd 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-25T03:38:27,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411257edc0726dd3546e8873ecf5d7cf0ea72_629be9b3b266d56b676cd5a9595f19cd is 50, key is test_row_0/A:col10/1732505907610/Put/seqid=0 2024-11-25T03:38:27,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742152_1328 (size=14594) 2024-11-25T03:38:27,639 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:27,646 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411257edc0726dd3546e8873ecf5d7cf0ea72_629be9b3b266d56b676cd5a9595f19cd to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411257edc0726dd3546e8873ecf5d7cf0ea72_629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:27,646 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/e59bc0c6c84d41028a65e8140df5d86c, store: [table=TestAcidGuarantees family=A region=629be9b3b266d56b676cd5a9595f19cd] 2024-11-25T03:38:27,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/e59bc0c6c84d41028a65e8140df5d86c is 175, key is test_row_0/A:col10/1732505907610/Put/seqid=0 2024-11-25T03:38:27,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742153_1329 (size=39549) 2024-11-25T03:38:27,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:27,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505967688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:27,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:27,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505967694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:27,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:27,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:27,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:27,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:27,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:27,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:27,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:27,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505967696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:27,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:27,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:27,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:27,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:27,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36696 deadline: 1732505967697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:27,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:27,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:27,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:27,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-25T03:38:27,758 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:38:27,759 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:38:27,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-25T03:38:27,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-25T03:38:27,761 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:38:27,762 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:38:27,762 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:38:27,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:27,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505967807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:27,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:27,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:27,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:27,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:27,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505967807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:27,817 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:27,817 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:27,817 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:27,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:27,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36696 deadline: 1732505967818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:27,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:27,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:27,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:27,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505967818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:27,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:27,830 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:27,830 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:27,830 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:27,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-25T03:38:27,914 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-25T03:38:27,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:27,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. as already flushing 2024-11-25T03:38:27,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:27,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:27,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:27,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:28,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:28,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36760 deadline: 1732505967997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:28,007 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,34881,1732505806588 for 'test_row_1' in 629be9b3b266d56b676cd5a9595f19cd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:38:28,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:28,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:28,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:28,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:28,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505968028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:28,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:28,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505968028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:28,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:28,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:28,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:28,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:28,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:28,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:28,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36696 deadline: 1732505968037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:28,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:28,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:28,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505968037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:28,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:28,046 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:28,046 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:28,046 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:28,059 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/e59bc0c6c84d41028a65e8140df5d86c 2024-11-25T03:38:28,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/fb4a9b73dc5b422f893a64013d35206c is 50, key is test_row_0/B:col10/1732505907610/Put/seqid=0 2024-11-25T03:38:28,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-25T03:38:28,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:28,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. as already flushing 2024-11-25T03:38:28,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:28,068 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:28,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:28,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:28,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742154_1330 (size=12001) 2024-11-25T03:38:28,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-25T03:38:28,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-25T03:38:28,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:28,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. as already flushing 2024-11-25T03:38:28,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:28,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:28,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:28,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:28,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:28,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505968337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:28,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:28,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:28,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505968338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:28,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:28,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:28,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:28,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:28,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:28,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:28,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36696 deadline: 1732505968357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:28,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:28,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:28,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505968357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:28,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:28,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:28,368 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:28,368 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:28,368 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:28,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-25T03:38:28,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:28,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. as already flushing 2024-11-25T03:38:28,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:28,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:28,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:28,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:28,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-25T03:38:28,472 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/fb4a9b73dc5b422f893a64013d35206c 2024-11-25T03:38:28,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/2256708ac72f4d189ab6dff8bd6d1c89 is 50, key is test_row_0/C:col10/1732505907610/Put/seqid=0 2024-11-25T03:38:28,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742155_1331 (size=12001) 2024-11-25T03:38:28,486 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/2256708ac72f4d189ab6dff8bd6d1c89 2024-11-25T03:38:28,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/e59bc0c6c84d41028a65e8140df5d86c as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/e59bc0c6c84d41028a65e8140df5d86c 2024-11-25T03:38:28,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/e59bc0c6c84d41028a65e8140df5d86c, entries=200, sequenceid=52, filesize=38.6 K 2024-11-25T03:38:28,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/fb4a9b73dc5b422f893a64013d35206c as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/fb4a9b73dc5b422f893a64013d35206c 2024-11-25T03:38:28,498 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/fb4a9b73dc5b422f893a64013d35206c, entries=150, sequenceid=52, filesize=11.7 K 2024-11-25T03:38:28,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/2256708ac72f4d189ab6dff8bd6d1c89 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/2256708ac72f4d189ab6dff8bd6d1c89 2024-11-25T03:38:28,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/2256708ac72f4d189ab6dff8bd6d1c89, entries=150, sequenceid=52, filesize=11.7 K 2024-11-25T03:38:28,503 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 629be9b3b266d56b676cd5a9595f19cd in 892ms, sequenceid=52, compaction requested=true 2024-11-25T03:38:28,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 629be9b3b266d56b676cd5a9595f19cd: 2024-11-25T03:38:28,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 629be9b3b266d56b676cd5a9595f19cd:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:38:28,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:38:28,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 629be9b3b266d56b676cd5a9595f19cd:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:38:28,503 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:38:28,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:38:28,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 629be9b3b266d56b676cd5a9595f19cd:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:38:28,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:38:28,503 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:38:28,504 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:38:28,504 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:38:28,505 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 629be9b3b266d56b676cd5a9595f19cd/B is initiating minor compaction (all files) 2024-11-25T03:38:28,505 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): 629be9b3b266d56b676cd5a9595f19cd/A is initiating minor compaction (all files) 2024-11-25T03:38:28,505 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 629be9b3b266d56b676cd5a9595f19cd/B in TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:28,505 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/b474b691a1cb4e8883d8511a7ecdd6ca, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/5bddac04ab7f448f85f14f3dd352b6e5, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/fb4a9b73dc5b422f893a64013d35206c] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp, totalSize=35.2 K 2024-11-25T03:38:28,505 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting b474b691a1cb4e8883d8511a7ecdd6ca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732505905628 2024-11-25T03:38:28,505 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bddac04ab7f448f85f14f3dd352b6e5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732505905785 2024-11-25T03:38:28,506 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 629be9b3b266d56b676cd5a9595f19cd/A in TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:28,506 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting fb4a9b73dc5b422f893a64013d35206c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732505906453 2024-11-25T03:38:28,506 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/12fecffaebcd49cc8dc9d9f003bc5f70, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/3772b5eaa7944a7394b50d74cf51964d, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/e59bc0c6c84d41028a65e8140df5d86c] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp, totalSize=107.5 K 2024-11-25T03:38:28,506 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:28,506 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. files: [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/12fecffaebcd49cc8dc9d9f003bc5f70, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/3772b5eaa7944a7394b50d74cf51964d, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/e59bc0c6c84d41028a65e8140df5d86c] 2024-11-25T03:38:28,506 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12fecffaebcd49cc8dc9d9f003bc5f70, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732505905628 2024-11-25T03:38:28,507 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3772b5eaa7944a7394b50d74cf51964d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732505905785 2024-11-25T03:38:28,508 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting e59bc0c6c84d41028a65e8140df5d86c, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732505906452 2024-11-25T03:38:28,513 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 629be9b3b266d56b676cd5a9595f19cd#B#compaction#281 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:38:28,514 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/21b4e9c1f38d4f71ad21e87c482e61f5 is 50, key is test_row_0/B:col10/1732505907610/Put/seqid=0 2024-11-25T03:38:28,517 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=629be9b3b266d56b676cd5a9595f19cd] 2024-11-25T03:38:28,521 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125481c5112684e442ba8d733706a1e2356_629be9b3b266d56b676cd5a9595f19cd store=[table=TestAcidGuarantees family=A region=629be9b3b266d56b676cd5a9595f19cd] 2024-11-25T03:38:28,523 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125481c5112684e442ba8d733706a1e2356_629be9b3b266d56b676cd5a9595f19cd, store=[table=TestAcidGuarantees family=A region=629be9b3b266d56b676cd5a9595f19cd] 2024-11-25T03:38:28,523 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125481c5112684e442ba8d733706a1e2356_629be9b3b266d56b676cd5a9595f19cd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=629be9b3b266d56b676cd5a9595f19cd] 2024-11-25T03:38:28,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-25T03:38:28,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:28,526 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 629be9b3b266d56b676cd5a9595f19cd 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-25T03:38:28,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742156_1332 (size=12104) 2024-11-25T03:38:28,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411252a5930f528cf4312ba96718ab7abcc87_629be9b3b266d56b676cd5a9595f19cd is 50, key is test_row_0/A:col10/1732505907694/Put/seqid=0 2024-11-25T03:38:28,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742157_1333 (size=4469) 2024-11-25T03:38:28,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742158_1334 (size=12154) 2024-11-25T03:38:28,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:28,556 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411252a5930f528cf4312ba96718ab7abcc87_629be9b3b266d56b676cd5a9595f19cd to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411252a5930f528cf4312ba96718ab7abcc87_629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:28,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/ec2e9320a04a40f2a906b7addb7ee82e, store: [table=TestAcidGuarantees family=A region=629be9b3b266d56b676cd5a9595f19cd] 2024-11-25T03:38:28,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/ec2e9320a04a40f2a906b7addb7ee82e is 175, key is test_row_0/A:col10/1732505907694/Put/seqid=0 2024-11-25T03:38:28,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742159_1335 (size=30955) 2024-11-25T03:38:28,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:28,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. as already flushing 2024-11-25T03:38:28,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:28,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505968878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:28,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:28,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:28,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:28,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:28,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505968884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:28,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:28,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:28,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:28,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:28,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36696 deadline: 1732505968886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:28,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:28,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:28,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:28,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-25T03:38:28,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:28,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505968888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:28,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:28,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:28,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:28,937 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/21b4e9c1f38d4f71ad21e87c482e61f5 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/21b4e9c1f38d4f71ad21e87c482e61f5 2024-11-25T03:38:28,942 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 629be9b3b266d56b676cd5a9595f19cd/B of 629be9b3b266d56b676cd5a9595f19cd into 21b4e9c1f38d4f71ad21e87c482e61f5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:38:28,942 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 629be9b3b266d56b676cd5a9595f19cd: 2024-11-25T03:38:28,942 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., storeName=629be9b3b266d56b676cd5a9595f19cd/B, priority=13, startTime=1732505908503; duration=0sec 2024-11-25T03:38:28,942 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:38:28,942 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 629be9b3b266d56b676cd5a9595f19cd:B 2024-11-25T03:38:28,942 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:38:28,943 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 629be9b3b266d56b676cd5a9595f19cd#A#compaction#282 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:38:28,944 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/814f1a81d07d49259070358a64ec96d9 is 175, key is test_row_0/A:col10/1732505907610/Put/seqid=0 2024-11-25T03:38:28,945 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:38:28,945 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 629be9b3b266d56b676cd5a9595f19cd/C is initiating minor compaction (all files) 2024-11-25T03:38:28,945 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 629be9b3b266d56b676cd5a9595f19cd/C in TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:28,945 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/c3e4165eb99643639fdb5496dceee90b, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/9ff74abb6f5d4f67ad7f793b0a27e727, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/2256708ac72f4d189ab6dff8bd6d1c89] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp, totalSize=35.2 K 2024-11-25T03:38:28,945 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting c3e4165eb99643639fdb5496dceee90b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732505905628 2024-11-25T03:38:28,946 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ff74abb6f5d4f67ad7f793b0a27e727, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732505905785 2024-11-25T03:38:28,946 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 2256708ac72f4d189ab6dff8bd6d1c89, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732505906453 2024-11-25T03:38:28,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742160_1336 (size=31058) 2024-11-25T03:38:28,954 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 629be9b3b266d56b676cd5a9595f19cd#C#compaction#284 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:38:28,955 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/fc7ce2ea177647a1bb9eb734f907e63e is 50, key is test_row_0/C:col10/1732505907610/Put/seqid=0 2024-11-25T03:38:28,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742161_1337 (size=12104) 2024-11-25T03:38:28,962 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/814f1a81d07d49259070358a64ec96d9 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/814f1a81d07d49259070358a64ec96d9 2024-11-25T03:38:28,967 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/ec2e9320a04a40f2a906b7addb7ee82e 2024-11-25T03:38:28,967 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 629be9b3b266d56b676cd5a9595f19cd/A of 629be9b3b266d56b676cd5a9595f19cd into 814f1a81d07d49259070358a64ec96d9(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:38:28,967 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 629be9b3b266d56b676cd5a9595f19cd: 2024-11-25T03:38:28,967 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., storeName=629be9b3b266d56b676cd5a9595f19cd/A, priority=13, startTime=1732505908503; duration=0sec 2024-11-25T03:38:28,967 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:38:28,967 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 629be9b3b266d56b676cd5a9595f19cd:A 2024-11-25T03:38:28,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/65ed1de3885a4a2f992de81ba1d63c83 is 50, key is test_row_0/B:col10/1732505907694/Put/seqid=0 2024-11-25T03:38:28,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742162_1338 (size=12001) 2024-11-25T03:38:28,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:28,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505968997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:28,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:28,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:28,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:28,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505968997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:28,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:28,999 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:28,999 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:28,999 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:29,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:29,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36696 deadline: 1732505969007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:29,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:29,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:29,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505969007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:29,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:29,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:29,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:29,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:29,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:29,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:29,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505969207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:29,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:29,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:29,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:29,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505969207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:29,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:29,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:29,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:29,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:29,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:29,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36696 deadline: 1732505969217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:29,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:29,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505969217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:29,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:29,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:29,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:29,226 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:29,226 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:29,226 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:29,366 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/fc7ce2ea177647a1bb9eb734f907e63e as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/fc7ce2ea177647a1bb9eb734f907e63e 2024-11-25T03:38:29,372 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 629be9b3b266d56b676cd5a9595f19cd/C of 629be9b3b266d56b676cd5a9595f19cd into fc7ce2ea177647a1bb9eb734f907e63e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:38:29,372 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 629be9b3b266d56b676cd5a9595f19cd: 2024-11-25T03:38:29,372 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., storeName=629be9b3b266d56b676cd5a9595f19cd/C, priority=13, startTime=1732505908503; duration=0sec 2024-11-25T03:38:29,372 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:38:29,372 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 629be9b3b266d56b676cd5a9595f19cd:C 2024-11-25T03:38:29,379 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/65ed1de3885a4a2f992de81ba1d63c83 2024-11-25T03:38:29,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/e325b1051af64b1bb0056307ab3b46a7 is 50, key is test_row_0/C:col10/1732505907694/Put/seqid=0 2024-11-25T03:38:29,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742163_1339 (size=12001) 2024-11-25T03:38:29,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:29,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505969527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:29,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:29,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:29,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505969527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:29,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:29,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:29,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:29,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:29,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:29,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:29,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36696 deadline: 1732505969537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:29,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:29,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505969537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:29,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:29,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:29,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:29,546 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:29,546 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:29,546 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:29,802 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/e325b1051af64b1bb0056307ab3b46a7 2024-11-25T03:38:29,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/ec2e9320a04a40f2a906b7addb7ee82e as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/ec2e9320a04a40f2a906b7addb7ee82e 2024-11-25T03:38:29,814 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/ec2e9320a04a40f2a906b7addb7ee82e, entries=150, sequenceid=77, filesize=30.2 K 2024-11-25T03:38:29,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/65ed1de3885a4a2f992de81ba1d63c83 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/65ed1de3885a4a2f992de81ba1d63c83 2024-11-25T03:38:29,819 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/65ed1de3885a4a2f992de81ba1d63c83, entries=150, sequenceid=77, filesize=11.7 K 2024-11-25T03:38:29,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/e325b1051af64b1bb0056307ab3b46a7 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/e325b1051af64b1bb0056307ab3b46a7 2024-11-25T03:38:29,825 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/e325b1051af64b1bb0056307ab3b46a7, entries=150, sequenceid=77, filesize=11.7 K 2024-11-25T03:38:29,826 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 629be9b3b266d56b676cd5a9595f19cd in 1301ms, sequenceid=77, compaction requested=false 2024-11-25T03:38:29,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 629be9b3b266d56b676cd5a9595f19cd: 2024-11-25T03:38:29,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:29,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-25T03:38:29,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-25T03:38:29,829 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-25T03:38:29,830 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0660 sec 2024-11-25T03:38:29,831 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 2.0710 sec 2024-11-25T03:38:29,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-25T03:38:29,908 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:38:29,908 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:38:29,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-25T03:38:29,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-25T03:38:29,913 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:38:29,914 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:38:29,914 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:38:30,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-25T03:38:30,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:30,022 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:38:30,022 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 629be9b3b266d56b676cd5a9595f19cd 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-25T03:38:30,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125421b312bb1524015b01a4334a433d3e1_629be9b3b266d56b676cd5a9595f19cd is 50, key is test_row_0/A:col10/1732505908885/Put/seqid=0 2024-11-25T03:38:30,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742164_1340 (size=14594) 2024-11-25T03:38:30,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-25T03:38:30,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:30,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. as already flushing 2024-11-25T03:38:30,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:30,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:30,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:30,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:30,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:30,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505970107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:30,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:30,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:30,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:30,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:30,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36696 deadline: 1732505970122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:30,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:30,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:30,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:30,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505970122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:30,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:30,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:30,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:30,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:30,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:30,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505970123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:30,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:30,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:30,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:30,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:30,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36760 deadline: 1732505970125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:30,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:30,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:30,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:30,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-25T03:38:30,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:30,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. as already flushing 2024-11-25T03:38:30,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:30,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:30,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:30,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:30,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-25T03:38:30,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:30,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505970227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:30,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:30,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:30,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:30,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:30,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36696 deadline: 1732505970247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:30,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:30,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:30,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505970247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:30,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:30,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:30,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:30,255 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:30,255 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:30,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36760 deadline: 1732505970247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:30,255 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:30,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:30,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:30,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:30,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:30,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505970247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:30,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:30,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:30,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:30,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-25T03:38:30,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:30,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. as already flushing 2024-11-25T03:38:30,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:30,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:30,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:30,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:30,453 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:30,457 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125421b312bb1524015b01a4334a433d3e1_629be9b3b266d56b676cd5a9595f19cd to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125421b312bb1524015b01a4334a433d3e1_629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:30,458 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/389ad6a18aaa42ebbba86b7689ca017a, store: [table=TestAcidGuarantees family=A region=629be9b3b266d56b676cd5a9595f19cd] 2024-11-25T03:38:30,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:30,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505970448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:30,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:30,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:30,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/389ad6a18aaa42ebbba86b7689ca017a is 175, key is test_row_0/A:col10/1732505908885/Put/seqid=0 2024-11-25T03:38:30,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:30,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:30,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505970457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:30,460 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:30,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:30,460 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:30,460 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:30,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505970457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:30,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:30,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:30,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:30,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742165_1341 (size=39549) 2024-11-25T03:38:30,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:30,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36696 deadline: 1732505970467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:30,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:30,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:30,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36760 deadline: 1732505970467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:30,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:30,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:30,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:30,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:30,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:30,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-25T03:38:30,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:30,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. as already flushing 2024-11-25T03:38:30,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:30,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:30,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:30,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:30,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-25T03:38:30,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-25T03:38:30,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:30,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. as already flushing 2024-11-25T03:38:30,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:30,687 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:30,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:30,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:30,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:30,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505970767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:30,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:30,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:30,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505970767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:30,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:30,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:30,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:30,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505970767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:30,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:30,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:30,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:30,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:30,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:30,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:30,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:30,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36696 deadline: 1732505970777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:30,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:30,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:30,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:30,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:30,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36760 deadline: 1732505970787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:30,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:30,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:30,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:30,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-25T03:38:30,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:30,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. as already flushing 2024-11-25T03:38:30,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:30,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:30,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:30,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:30,864 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=92, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/389ad6a18aaa42ebbba86b7689ca017a 2024-11-25T03:38:30,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/79676ba569a24d26876a783323473788 is 50, key is test_row_0/B:col10/1732505908885/Put/seqid=0 2024-11-25T03:38:30,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742166_1342 (size=12001) 2024-11-25T03:38:30,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-25T03:38:30,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:30,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. as already flushing 2024-11-25T03:38:30,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:30,995 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:30,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:31,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:31,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-25T03:38:31,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-25T03:38:31,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:31,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. as already flushing 2024-11-25T03:38:31,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:31,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:31,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:31,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:31,278 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/79676ba569a24d26876a783323473788 2024-11-25T03:38:31,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:31,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505971277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:31,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/2b65c52e9fe14a4db1735e8c68b74c25 is 50, key is test_row_0/C:col10/1732505908885/Put/seqid=0 2024-11-25T03:38:31,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:31,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:31,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:31,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:31,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505971288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:31,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:31,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:31,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:31,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505971288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:31,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:31,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:31,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:31,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:31,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742167_1343 (size=12001) 2024-11-25T03:38:31,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-25T03:38:31,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:31,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. as already flushing 2024-11-25T03:38:31,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:31,306 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:31,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:31,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:31,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:31,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36696 deadline: 1732505971298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:31,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:31,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:31,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:31,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:31,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36760 deadline: 1732505971307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:31,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:31,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:31,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:31,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-25T03:38:31,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:31,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. as already flushing 2024-11-25T03:38:31,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:31,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:31,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:31,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:31,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-25T03:38:31,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:31,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. as already flushing 2024-11-25T03:38:31,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:31,612 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:31,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:31,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:31,695 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/2b65c52e9fe14a4db1735e8c68b74c25 2024-11-25T03:38:31,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/389ad6a18aaa42ebbba86b7689ca017a as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/389ad6a18aaa42ebbba86b7689ca017a 2024-11-25T03:38:31,704 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/389ad6a18aaa42ebbba86b7689ca017a, entries=200, sequenceid=92, filesize=38.6 K 2024-11-25T03:38:31,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/79676ba569a24d26876a783323473788 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/79676ba569a24d26876a783323473788 2024-11-25T03:38:31,708 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/79676ba569a24d26876a783323473788, entries=150, sequenceid=92, filesize=11.7 K 2024-11-25T03:38:31,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/2b65c52e9fe14a4db1735e8c68b74c25 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/2b65c52e9fe14a4db1735e8c68b74c25 2024-11-25T03:38:31,714 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/2b65c52e9fe14a4db1735e8c68b74c25, entries=150, sequenceid=92, filesize=11.7 K 2024-11-25T03:38:31,715 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 629be9b3b266d56b676cd5a9595f19cd in 1692ms, sequenceid=92, compaction requested=true 2024-11-25T03:38:31,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 629be9b3b266d56b676cd5a9595f19cd: 2024-11-25T03:38:31,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 629be9b3b266d56b676cd5a9595f19cd:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:38:31,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:38:31,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 629be9b3b266d56b676cd5a9595f19cd:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:38:31,715 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:38:31,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:38:31,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 629be9b3b266d56b676cd5a9595f19cd:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:38:31,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:38:31,715 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:38:31,716 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:38:31,716 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:38:31,716 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): 629be9b3b266d56b676cd5a9595f19cd/A is initiating minor compaction (all files) 2024-11-25T03:38:31,716 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 629be9b3b266d56b676cd5a9595f19cd/B is initiating minor compaction (all files) 2024-11-25T03:38:31,716 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 629be9b3b266d56b676cd5a9595f19cd/B in TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:31,716 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 629be9b3b266d56b676cd5a9595f19cd/A in TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:31,716 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/814f1a81d07d49259070358a64ec96d9, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/ec2e9320a04a40f2a906b7addb7ee82e, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/389ad6a18aaa42ebbba86b7689ca017a] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp, totalSize=99.2 K 2024-11-25T03:38:31,716 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/21b4e9c1f38d4f71ad21e87c482e61f5, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/65ed1de3885a4a2f992de81ba1d63c83, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/79676ba569a24d26876a783323473788] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp, totalSize=35.3 K 2024-11-25T03:38:31,716 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:31,716 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. files: [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/814f1a81d07d49259070358a64ec96d9, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/ec2e9320a04a40f2a906b7addb7ee82e, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/389ad6a18aaa42ebbba86b7689ca017a] 2024-11-25T03:38:31,716 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 814f1a81d07d49259070358a64ec96d9, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732505906453 2024-11-25T03:38:31,716 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 21b4e9c1f38d4f71ad21e87c482e61f5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732505906453 2024-11-25T03:38:31,717 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec2e9320a04a40f2a906b7addb7ee82e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732505907692 2024-11-25T03:38:31,717 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 65ed1de3885a4a2f992de81ba1d63c83, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732505907692 2024-11-25T03:38:31,717 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 389ad6a18aaa42ebbba86b7689ca017a, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732505908877 2024-11-25T03:38:31,717 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 79676ba569a24d26876a783323473788, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732505908883 2024-11-25T03:38:31,723 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=629be9b3b266d56b676cd5a9595f19cd] 2024-11-25T03:38:31,725 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 629be9b3b266d56b676cd5a9595f19cd#B#compaction#291 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:38:31,726 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/1028e2447cfd48ada0057579afd0dedd is 50, key is test_row_0/B:col10/1732505908885/Put/seqid=0 2024-11-25T03:38:31,728 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112504b1f0b7b39f4e229e3f7403617beaac_629be9b3b266d56b676cd5a9595f19cd store=[table=TestAcidGuarantees family=A region=629be9b3b266d56b676cd5a9595f19cd] 2024-11-25T03:38:31,729 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112504b1f0b7b39f4e229e3f7403617beaac_629be9b3b266d56b676cd5a9595f19cd, store=[table=TestAcidGuarantees family=A region=629be9b3b266d56b676cd5a9595f19cd] 2024-11-25T03:38:31,729 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112504b1f0b7b39f4e229e3f7403617beaac_629be9b3b266d56b676cd5a9595f19cd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=629be9b3b266d56b676cd5a9595f19cd] 2024-11-25T03:38:31,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742168_1344 (size=12207) 2024-11-25T03:38:31,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742169_1345 (size=4469) 2024-11-25T03:38:31,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-25T03:38:31,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:31,765 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 629be9b3b266d56b676cd5a9595f19cd 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-25T03:38:31,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112504d5f24fc0064a968546939b5c1fd145_629be9b3b266d56b676cd5a9595f19cd is 50, key is test_row_0/A:col10/1732505910118/Put/seqid=0 2024-11-25T03:38:31,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742170_1346 (size=12154) 2024-11-25T03:38:32,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-25T03:38:32,139 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 629be9b3b266d56b676cd5a9595f19cd#A#compaction#290 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:38:32,139 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/1028e2447cfd48ada0057579afd0dedd as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/1028e2447cfd48ada0057579afd0dedd 2024-11-25T03:38:32,140 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/bd9dd7f21ba34c778b9c0ec6d4dd620f is 175, key is test_row_0/A:col10/1732505908885/Put/seqid=0 2024-11-25T03:38:32,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742171_1347 (size=31161) 2024-11-25T03:38:32,146 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 629be9b3b266d56b676cd5a9595f19cd/B of 629be9b3b266d56b676cd5a9595f19cd into 1028e2447cfd48ada0057579afd0dedd(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:38:32,147 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 629be9b3b266d56b676cd5a9595f19cd: 2024-11-25T03:38:32,147 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., storeName=629be9b3b266d56b676cd5a9595f19cd/B, priority=13, startTime=1732505911715; duration=0sec 2024-11-25T03:38:32,147 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:38:32,147 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 629be9b3b266d56b676cd5a9595f19cd:B 2024-11-25T03:38:32,147 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:38:32,148 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:38:32,148 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 629be9b3b266d56b676cd5a9595f19cd/C is initiating minor compaction (all files) 2024-11-25T03:38:32,148 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 629be9b3b266d56b676cd5a9595f19cd/C in TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:32,148 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/fc7ce2ea177647a1bb9eb734f907e63e, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/e325b1051af64b1bb0056307ab3b46a7, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/2b65c52e9fe14a4db1735e8c68b74c25] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp, totalSize=35.3 K 2024-11-25T03:38:32,148 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting fc7ce2ea177647a1bb9eb734f907e63e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732505906453 2024-11-25T03:38:32,149 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting e325b1051af64b1bb0056307ab3b46a7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732505907692 2024-11-25T03:38:32,149 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b65c52e9fe14a4db1735e8c68b74c25, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732505908883 2024-11-25T03:38:32,151 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/bd9dd7f21ba34c778b9c0ec6d4dd620f as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/bd9dd7f21ba34c778b9c0ec6d4dd620f 2024-11-25T03:38:32,157 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 629be9b3b266d56b676cd5a9595f19cd/A of 629be9b3b266d56b676cd5a9595f19cd into bd9dd7f21ba34c778b9c0ec6d4dd620f(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:38:32,157 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 629be9b3b266d56b676cd5a9595f19cd: 2024-11-25T03:38:32,157 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., storeName=629be9b3b266d56b676cd5a9595f19cd/A, priority=13, startTime=1732505911715; duration=0sec 2024-11-25T03:38:32,157 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:38:32,157 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 629be9b3b266d56b676cd5a9595f19cd:A 2024-11-25T03:38:32,157 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 629be9b3b266d56b676cd5a9595f19cd#C#compaction#293 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:38:32,158 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/682720e13a0f4f03acfd406a85cde4c8 is 50, key is test_row_0/C:col10/1732505908885/Put/seqid=0 2024-11-25T03:38:32,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742172_1348 (size=12207) 2024-11-25T03:38:32,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:32,181 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112504d5f24fc0064a968546939b5c1fd145_629be9b3b266d56b676cd5a9595f19cd to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112504d5f24fc0064a968546939b5c1fd145_629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:32,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/12cb4e7b51e74f9da84cb44d6f911263, store: [table=TestAcidGuarantees family=A region=629be9b3b266d56b676cd5a9595f19cd] 2024-11-25T03:38:32,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/12cb4e7b51e74f9da84cb44d6f911263 is 175, key is test_row_0/A:col10/1732505910118/Put/seqid=0 2024-11-25T03:38:32,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742173_1349 (size=30955) 2024-11-25T03:38:32,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. as already flushing 2024-11-25T03:38:32,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:32,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:32,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36696 deadline: 1732505972318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:32,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:32,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505972320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:32,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:32,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505972323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:32,329 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,34881,1732505806588 for 'test_row_1' in 629be9b3b266d56b676cd5a9595f19cd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:38:32,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:32,329 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:32,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:32,329 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:32,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:32,329 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:32,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:32,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:32,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:32,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:32,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36760 deadline: 1732505972327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:32,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:32,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505972328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:32,336 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,34881,1732505806588 for 'test_row_1' in 629be9b3b266d56b676cd5a9595f19cd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:38:32,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:32,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:32,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:32,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:32,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:32,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:32,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:32,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505972437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:32,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:32,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:32,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:32,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505972437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:32,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:32,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:32,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:32,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:32,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:32,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505972447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:32,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:32,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:32,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:32,567 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/682720e13a0f4f03acfd406a85cde4c8 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/682720e13a0f4f03acfd406a85cde4c8 2024-11-25T03:38:32,577 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 629be9b3b266d56b676cd5a9595f19cd/C of 629be9b3b266d56b676cd5a9595f19cd into 682720e13a0f4f03acfd406a85cde4c8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:38:32,577 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 629be9b3b266d56b676cd5a9595f19cd: 2024-11-25T03:38:32,577 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., storeName=629be9b3b266d56b676cd5a9595f19cd/C, priority=13, startTime=1732505911715; duration=0sec 2024-11-25T03:38:32,577 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:38:32,577 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 629be9b3b266d56b676cd5a9595f19cd:C 2024-11-25T03:38:32,589 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/12cb4e7b51e74f9da84cb44d6f911263 2024-11-25T03:38:32,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/55727c6063034330a5fbc06cb8b2a991 is 50, key is test_row_0/B:col10/1732505910118/Put/seqid=0 2024-11-25T03:38:32,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742174_1350 (size=12001) 2024-11-25T03:38:32,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:32,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505972648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:32,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:32,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505972648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:32,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:32,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:32,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:32,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:32,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:32,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:32,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:32,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505972667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:32,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:32,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:32,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:32,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:32,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505972967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:32,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:32,977 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:32,977 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:32,977 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:32,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505972967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:32,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:32,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:32,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:32,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:32,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505972978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:32,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:32,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:32,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:33,001 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/55727c6063034330a5fbc06cb8b2a991 2024-11-25T03:38:33,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/56e2e7ab20ee463a9d21e5242871ad44 is 50, key is test_row_0/C:col10/1732505910118/Put/seqid=0 2024-11-25T03:38:33,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742175_1351 (size=12001) 2024-11-25T03:38:33,413 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/56e2e7ab20ee463a9d21e5242871ad44 2024-11-25T03:38:33,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/12cb4e7b51e74f9da84cb44d6f911263 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/12cb4e7b51e74f9da84cb44d6f911263 2024-11-25T03:38:33,421 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/12cb4e7b51e74f9da84cb44d6f911263, entries=150, sequenceid=116, filesize=30.2 K 2024-11-25T03:38:33,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/55727c6063034330a5fbc06cb8b2a991 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/55727c6063034330a5fbc06cb8b2a991 2024-11-25T03:38:33,427 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/55727c6063034330a5fbc06cb8b2a991, entries=150, sequenceid=116, filesize=11.7 K 2024-11-25T03:38:33,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/56e2e7ab20ee463a9d21e5242871ad44 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/56e2e7ab20ee463a9d21e5242871ad44 2024-11-25T03:38:33,431 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/56e2e7ab20ee463a9d21e5242871ad44, entries=150, sequenceid=116, filesize=11.7 K 2024-11-25T03:38:33,432 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 629be9b3b266d56b676cd5a9595f19cd in 1667ms, sequenceid=116, compaction requested=false 2024-11-25T03:38:33,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 629be9b3b266d56b676cd5a9595f19cd: 2024-11-25T03:38:33,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:33,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-25T03:38:33,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-25T03:38:33,440 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-25T03:38:33,440 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5190 sec 2024-11-25T03:38:33,442 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 3.5330 sec 2024-11-25T03:38:33,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:33,493 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:38:33,493 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 629be9b3b266d56b676cd5a9595f19cd 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-25T03:38:33,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125be9031e7a9f94ec6bf0297878b8fbfe1_629be9b3b266d56b676cd5a9595f19cd is 50, key is test_row_0/A:col10/1732505913491/Put/seqid=0 2024-11-25T03:38:33,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742176_1352 (size=17184) 2024-11-25T03:38:33,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:33,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505973554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:33,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:33,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:33,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:33,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:33,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505973558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:33,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:33,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:33,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:33,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:33,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505973559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:33,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:33,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:33,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:33,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:33,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505973667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:33,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:33,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:33,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:33,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:33,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505973667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:33,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:33,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505973677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:33,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:33,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:33,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:33,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:33,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505973877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:33,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:33,887 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:33,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505973877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:33,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:33,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:33,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:33,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:33,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:33,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:33,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505973887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:33,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:33,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:33,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:33,908 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:33,912 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125be9031e7a9f94ec6bf0297878b8fbfe1_629be9b3b266d56b676cd5a9595f19cd to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125be9031e7a9f94ec6bf0297878b8fbfe1_629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:33,913 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/36813e12e0974dc09bb555701652cfd9, store: [table=TestAcidGuarantees family=A region=629be9b3b266d56b676cd5a9595f19cd] 2024-11-25T03:38:33,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/36813e12e0974dc09bb555701652cfd9 is 175, key is test_row_0/A:col10/1732505913491/Put/seqid=0 2024-11-25T03:38:33,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742177_1353 (size=48289) 2024-11-25T03:38:34,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-25T03:38:34,068 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:38:34,068 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:38:34,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-25T03:38:34,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-25T03:38:34,070 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:38:34,071 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:38:34,071 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:38:34,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-25T03:38:34,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:34,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505974198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:34,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:34,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:34,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505974198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:34,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:34,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:34,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:34,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:34,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505974198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:34,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:34,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:34,206 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:34,206 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:34,206 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:34,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-25T03:38:34,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:34,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. as already flushing 2024-11-25T03:38:34,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:34,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:34,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:34,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:34,318 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/36813e12e0974dc09bb555701652cfd9 2024-11-25T03:38:34,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/57331c87d015481f9a152927af1962ef is 50, key is test_row_0/B:col10/1732505913491/Put/seqid=0 2024-11-25T03:38:34,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742178_1354 (size=12101) 2024-11-25T03:38:34,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:34,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36696 deadline: 1732505974357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:34,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:34,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36760 deadline: 1732505974357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:34,368 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,34881,1732505806588 for 'test_row_1' in 629be9b3b266d56b676cd5a9595f19cd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4246 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:38:34,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:34,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:34,369 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,34881,1732505806588 for 'test_row_1' in 629be9b3b266d56b676cd5a9595f19cd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4244 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:38:34,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:34,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:34,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:34,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:34,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-25T03:38:34,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:34,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. as already flushing 2024-11-25T03:38:34,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:34,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:34,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:34,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:34,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-25T03:38:34,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-25T03:38:34,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:34,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. as already flushing 2024-11-25T03:38:34,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:34,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:34,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:34,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:34,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-25T03:38:34,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:34,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. as already flushing 2024-11-25T03:38:34,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:34,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:34,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:34,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:34,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-25T03:38:34,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:34,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505974707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:34,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:34,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:34,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:34,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:34,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505974717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:34,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:34,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:34,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:34,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:34,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505974717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:34,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:34,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:34,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:34,728 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/57331c87d015481f9a152927af1962ef 2024-11-25T03:38:34,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/d62cf6bad0f346298cafe99f8fead3b6 is 50, key is test_row_0/C:col10/1732505913491/Put/seqid=0 2024-11-25T03:38:34,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742179_1355 (size=12101) 2024-11-25T03:38:34,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-25T03:38:34,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:34,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. as already flushing 2024-11-25T03:38:34,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:34,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:34,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:34,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:34,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-25T03:38:34,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:34,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. as already flushing 2024-11-25T03:38:34,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:34,987 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:34,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:34,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:35,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/d62cf6bad0f346298cafe99f8fead3b6 2024-11-25T03:38:35,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-25T03:38:35,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:35,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. as already flushing 2024-11-25T03:38:35,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:35,140 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:35,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:35,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:35,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/36813e12e0974dc09bb555701652cfd9 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/36813e12e0974dc09bb555701652cfd9 2024-11-25T03:38:35,149 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/36813e12e0974dc09bb555701652cfd9, entries=250, sequenceid=133, filesize=47.2 K 2024-11-25T03:38:35,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/57331c87d015481f9a152927af1962ef as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/57331c87d015481f9a152927af1962ef 2024-11-25T03:38:35,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/57331c87d015481f9a152927af1962ef, entries=150, sequenceid=133, filesize=11.8 K 2024-11-25T03:38:35,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/d62cf6bad0f346298cafe99f8fead3b6 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/d62cf6bad0f346298cafe99f8fead3b6 2024-11-25T03:38:35,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/d62cf6bad0f346298cafe99f8fead3b6, entries=150, sequenceid=133, filesize=11.8 K 2024-11-25T03:38:35,158 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 629be9b3b266d56b676cd5a9595f19cd in 1665ms, sequenceid=133, compaction requested=true 2024-11-25T03:38:35,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 629be9b3b266d56b676cd5a9595f19cd: 2024-11-25T03:38:35,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 629be9b3b266d56b676cd5a9595f19cd:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:38:35,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:38:35,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 629be9b3b266d56b676cd5a9595f19cd:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:38:35,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:38:35,158 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:38:35,158 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:38:35,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 629be9b3b266d56b676cd5a9595f19cd:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:38:35,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:38:35,160 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:38:35,160 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110405 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:38:35,160 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 629be9b3b266d56b676cd5a9595f19cd/B is initiating minor compaction (all files) 2024-11-25T03:38:35,160 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): 629be9b3b266d56b676cd5a9595f19cd/A is initiating minor compaction (all files) 2024-11-25T03:38:35,160 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 629be9b3b266d56b676cd5a9595f19cd/B in TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:35,160 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 629be9b3b266d56b676cd5a9595f19cd/A in TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:35,160 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/1028e2447cfd48ada0057579afd0dedd, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/55727c6063034330a5fbc06cb8b2a991, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/57331c87d015481f9a152927af1962ef] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp, totalSize=35.5 K 2024-11-25T03:38:35,160 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/bd9dd7f21ba34c778b9c0ec6d4dd620f, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/12cb4e7b51e74f9da84cb44d6f911263, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/36813e12e0974dc09bb555701652cfd9] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp, totalSize=107.8 K 2024-11-25T03:38:35,160 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:35,160 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. files: [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/bd9dd7f21ba34c778b9c0ec6d4dd620f, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/12cb4e7b51e74f9da84cb44d6f911263, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/36813e12e0974dc09bb555701652cfd9] 2024-11-25T03:38:35,161 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd9dd7f21ba34c778b9c0ec6d4dd620f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732505908883 2024-11-25T03:38:35,161 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 1028e2447cfd48ada0057579afd0dedd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732505908883 2024-11-25T03:38:35,161 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12cb4e7b51e74f9da84cb44d6f911263, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732505910118 2024-11-25T03:38:35,161 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 55727c6063034330a5fbc06cb8b2a991, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732505910118 2024-11-25T03:38:35,162 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36813e12e0974dc09bb555701652cfd9, keycount=250, bloomtype=ROW, size=47.2 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732505912319 2024-11-25T03:38:35,162 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 57331c87d015481f9a152927af1962ef, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732505912322 2024-11-25T03:38:35,173 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 629be9b3b266d56b676cd5a9595f19cd#B#compaction#299 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:38:35,173 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/d0aa8f6cc3e84f70b55c0c1991844808 is 50, key is test_row_0/B:col10/1732505913491/Put/seqid=0 2024-11-25T03:38:35,183 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=629be9b3b266d56b676cd5a9595f19cd] 2024-11-25T03:38:35,189 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125670e0dee89e34f58b18d74116117f7cc_629be9b3b266d56b676cd5a9595f19cd store=[table=TestAcidGuarantees family=A region=629be9b3b266d56b676cd5a9595f19cd] 2024-11-25T03:38:35,191 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125670e0dee89e34f58b18d74116117f7cc_629be9b3b266d56b676cd5a9595f19cd, store=[table=TestAcidGuarantees family=A region=629be9b3b266d56b676cd5a9595f19cd] 2024-11-25T03:38:35,191 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125670e0dee89e34f58b18d74116117f7cc_629be9b3b266d56b676cd5a9595f19cd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=629be9b3b266d56b676cd5a9595f19cd] 2024-11-25T03:38:35,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742180_1356 (size=12409) 2024-11-25T03:38:35,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742181_1357 (size=4469) 2024-11-25T03:38:35,204 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 629be9b3b266d56b676cd5a9595f19cd#A#compaction#300 average throughput is 1.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:38:35,205 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/e74f589ffac74e0cbb97a94e457a545a is 175, key is test_row_0/A:col10/1732505913491/Put/seqid=0 2024-11-25T03:38:35,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-25T03:38:35,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742182_1358 (size=31363) 2024-11-25T03:38:35,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-25T03:38:35,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:35,293 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 629be9b3b266d56b676cd5a9595f19cd 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-25T03:38:35,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125cf405a6833484c6c87af07054d7225f9_629be9b3b266d56b676cd5a9595f19cd is 50, key is test_row_0/A:col10/1732505913538/Put/seqid=0 2024-11-25T03:38:35,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742183_1359 (size=12304) 2024-11-25T03:38:35,606 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/d0aa8f6cc3e84f70b55c0c1991844808 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/d0aa8f6cc3e84f70b55c0c1991844808 2024-11-25T03:38:35,610 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 629be9b3b266d56b676cd5a9595f19cd/B of 629be9b3b266d56b676cd5a9595f19cd into d0aa8f6cc3e84f70b55c0c1991844808(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:38:35,610 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 629be9b3b266d56b676cd5a9595f19cd: 2024-11-25T03:38:35,610 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., storeName=629be9b3b266d56b676cd5a9595f19cd/B, priority=13, startTime=1732505915158; duration=0sec 2024-11-25T03:38:35,610 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:38:35,610 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 629be9b3b266d56b676cd5a9595f19cd:B 2024-11-25T03:38:35,610 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:38:35,611 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:38:35,611 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 629be9b3b266d56b676cd5a9595f19cd/C is initiating minor compaction (all files) 2024-11-25T03:38:35,613 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 629be9b3b266d56b676cd5a9595f19cd/C in TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:35,613 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/682720e13a0f4f03acfd406a85cde4c8, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/56e2e7ab20ee463a9d21e5242871ad44, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/d62cf6bad0f346298cafe99f8fead3b6] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp, totalSize=35.5 K 2024-11-25T03:38:35,613 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 682720e13a0f4f03acfd406a85cde4c8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732505908883 2024-11-25T03:38:35,614 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 56e2e7ab20ee463a9d21e5242871ad44, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732505910118 2024-11-25T03:38:35,616 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting d62cf6bad0f346298cafe99f8fead3b6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732505912322 2024-11-25T03:38:35,619 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/e74f589ffac74e0cbb97a94e457a545a as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/e74f589ffac74e0cbb97a94e457a545a 2024-11-25T03:38:35,624 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 629be9b3b266d56b676cd5a9595f19cd/A of 629be9b3b266d56b676cd5a9595f19cd into e74f589ffac74e0cbb97a94e457a545a(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:38:35,624 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 629be9b3b266d56b676cd5a9595f19cd: 2024-11-25T03:38:35,624 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., storeName=629be9b3b266d56b676cd5a9595f19cd/A, priority=13, startTime=1732505915158; duration=0sec 2024-11-25T03:38:35,624 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:38:35,624 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 629be9b3b266d56b676cd5a9595f19cd:A 2024-11-25T03:38:35,625 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 629be9b3b266d56b676cd5a9595f19cd#C#compaction#302 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:38:35,626 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/ebdfcf1e175f4987832e16dc4fea20ff is 50, key is test_row_0/C:col10/1732505913491/Put/seqid=0 2024-11-25T03:38:35,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742184_1360 (size=12409) 2024-11-25T03:38:35,633 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/ebdfcf1e175f4987832e16dc4fea20ff as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/ebdfcf1e175f4987832e16dc4fea20ff 2024-11-25T03:38:35,638 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 629be9b3b266d56b676cd5a9595f19cd/C of 629be9b3b266d56b676cd5a9595f19cd into ebdfcf1e175f4987832e16dc4fea20ff(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:38:35,638 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 629be9b3b266d56b676cd5a9595f19cd: 2024-11-25T03:38:35,638 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., storeName=629be9b3b266d56b676cd5a9595f19cd/C, priority=13, startTime=1732505915158; duration=0sec 2024-11-25T03:38:35,638 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:38:35,638 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 629be9b3b266d56b676cd5a9595f19cd:C 2024-11-25T03:38:35,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:35,708 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125cf405a6833484c6c87af07054d7225f9_629be9b3b266d56b676cd5a9595f19cd to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125cf405a6833484c6c87af07054d7225f9_629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:35,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/16e6cd22eadb4f9a90c49b3a9dafb837, store: [table=TestAcidGuarantees family=A region=629be9b3b266d56b676cd5a9595f19cd] 2024-11-25T03:38:35,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/16e6cd22eadb4f9a90c49b3a9dafb837 is 175, key is test_row_0/A:col10/1732505913538/Put/seqid=0 2024-11-25T03:38:35,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742185_1361 (size=31105) 2024-11-25T03:38:35,715 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/16e6cd22eadb4f9a90c49b3a9dafb837 2024-11-25T03:38:35,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/9e24bab4733c4ff4a3e0cb2ae0c959d4 is 50, key is test_row_0/B:col10/1732505913538/Put/seqid=0 2024-11-25T03:38:35,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742186_1362 (size=12151) 2024-11-25T03:38:35,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:35,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. as already flushing 2024-11-25T03:38:35,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:35,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505975754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505975755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:35,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:35,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:35,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505975758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:35,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:35,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:35,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:35,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:35,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:35,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:35,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:35,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:35,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505975878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:35,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:35,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:35,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:35,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505975878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:35,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:35,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:35,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:35,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:35,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:35,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505975878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:35,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:35,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:35,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:36,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:36,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505976097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:36,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:36,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:36,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:36,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:36,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505976098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:36,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:36,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505976098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:36,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:36,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:36,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:36,105 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:36,105 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:36,105 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:36,125 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/9e24bab4733c4ff4a3e0cb2ae0c959d4 2024-11-25T03:38:36,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/f6ab1f10e69e4b1e9f9d4e7e208c3973 is 50, key is test_row_0/C:col10/1732505913538/Put/seqid=0 2024-11-25T03:38:36,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742187_1363 (size=12151) 2024-11-25T03:38:36,146 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/f6ab1f10e69e4b1e9f9d4e7e208c3973 2024-11-25T03:38:36,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/16e6cd22eadb4f9a90c49b3a9dafb837 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/16e6cd22eadb4f9a90c49b3a9dafb837 2024-11-25T03:38:36,157 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/16e6cd22eadb4f9a90c49b3a9dafb837, entries=150, sequenceid=155, filesize=30.4 K 2024-11-25T03:38:36,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/9e24bab4733c4ff4a3e0cb2ae0c959d4 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/9e24bab4733c4ff4a3e0cb2ae0c959d4 2024-11-25T03:38:36,163 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/9e24bab4733c4ff4a3e0cb2ae0c959d4, entries=150, sequenceid=155, filesize=11.9 K 2024-11-25T03:38:36,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/f6ab1f10e69e4b1e9f9d4e7e208c3973 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/f6ab1f10e69e4b1e9f9d4e7e208c3973 2024-11-25T03:38:36,167 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/f6ab1f10e69e4b1e9f9d4e7e208c3973, entries=150, sequenceid=155, filesize=11.9 K 2024-11-25T03:38:36,168 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 629be9b3b266d56b676cd5a9595f19cd in 875ms, sequenceid=155, compaction requested=false 2024-11-25T03:38:36,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 629be9b3b266d56b676cd5a9595f19cd: 2024-11-25T03:38:36,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:36,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-25T03:38:36,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-25T03:38:36,172 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-25T03:38:36,172 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0990 sec 2024-11-25T03:38:36,174 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.1050 sec 2024-11-25T03:38:36,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-25T03:38:36,218 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:38:36,218 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:38:36,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-25T03:38:36,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-25T03:38:36,221 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:38:36,221 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:38:36,221 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:38:36,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-25T03:38:36,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-25T03:38:36,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:36,374 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 629be9b3b266d56b676cd5a9595f19cd 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-25T03:38:36,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125901635e9ad7e4705bc14c23c120ae216_629be9b3b266d56b676cd5a9595f19cd is 50, key is test_row_0/A:col10/1732505915757/Put/seqid=0 2024-11-25T03:38:36,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742188_1364 (size=12304) 2024-11-25T03:38:36,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:36,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. as already flushing 2024-11-25T03:38:36,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:36,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505976463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:36,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:36,475 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:36,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505976469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:36,475 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:36,475 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:36,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:36,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:36,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:36,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:36,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505976470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:36,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:36,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:36,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:36,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-25T03:38:36,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:36,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505976577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:36,580 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:36,580 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:36,580 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:36,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:36,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505976577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:36,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:36,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:36,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:36,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:36,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505976587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:36,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:36,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:36,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:36,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:36,790 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125901635e9ad7e4705bc14c23c120ae216_629be9b3b266d56b676cd5a9595f19cd to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125901635e9ad7e4705bc14c23c120ae216_629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:36,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/2950c32403e84d91b55aa3c9708544d7, store: [table=TestAcidGuarantees family=A region=629be9b3b266d56b676cd5a9595f19cd] 2024-11-25T03:38:36,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/2950c32403e84d91b55aa3c9708544d7 is 175, key is test_row_0/A:col10/1732505915757/Put/seqid=0 2024-11-25T03:38:36,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742189_1365 (size=31105) 2024-11-25T03:38:36,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:36,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505976788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:36,798 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:36,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:36,798 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:36,798 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:36,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505976788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:36,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:36,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:36,799 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/2950c32403e84d91b55aa3c9708544d7 2024-11-25T03:38:36,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:36,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:36,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505976797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:36,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:36,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:36,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:36,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/249a0a66cde5448aa5d9cd286462955e is 50, key is test_row_0/B:col10/1732505915757/Put/seqid=0 2024-11-25T03:38:36,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742190_1366 (size=12151) 2024-11-25T03:38:36,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-25T03:38:37,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:37,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505977108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:37,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:37,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505977108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:37,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:37,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:37,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:37,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:37,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:37,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:37,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:37,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505977117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:37,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:37,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:37,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:37,219 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/249a0a66cde5448aa5d9cd286462955e 2024-11-25T03:38:37,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/d1ce8015cee242f1b84f2757c258d85b is 50, key is test_row_0/C:col10/1732505915757/Put/seqid=0 2024-11-25T03:38:37,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742191_1367 (size=12151) 2024-11-25T03:38:37,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-25T03:38:37,631 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/d1ce8015cee242f1b84f2757c258d85b 2024-11-25T03:38:37,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/2950c32403e84d91b55aa3c9708544d7 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/2950c32403e84d91b55aa3c9708544d7 2024-11-25T03:38:37,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:37,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505977628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:37,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:37,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505977628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:37,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:37,639 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/2950c32403e84d91b55aa3c9708544d7, entries=150, sequenceid=172, filesize=30.4 K 2024-11-25T03:38:37,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505977628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:37,639 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:37,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:37,639 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:37,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:37,639 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:37,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:37,639 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:37,639 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:37,639 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:37,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/249a0a66cde5448aa5d9cd286462955e as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/249a0a66cde5448aa5d9cd286462955e 2024-11-25T03:38:37,643 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/249a0a66cde5448aa5d9cd286462955e, entries=150, sequenceid=172, filesize=11.9 K 2024-11-25T03:38:37,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/d1ce8015cee242f1b84f2757c258d85b as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/d1ce8015cee242f1b84f2757c258d85b 2024-11-25T03:38:37,648 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/d1ce8015cee242f1b84f2757c258d85b, entries=150, sequenceid=172, filesize=11.9 K 2024-11-25T03:38:37,648 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 629be9b3b266d56b676cd5a9595f19cd in 1274ms, sequenceid=172, compaction requested=true 2024-11-25T03:38:37,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 629be9b3b266d56b676cd5a9595f19cd: 2024-11-25T03:38:37,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:37,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-25T03:38:37,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-25T03:38:37,651 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-25T03:38:37,651 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4290 sec 2024-11-25T03:38:37,653 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.4330 sec 2024-11-25T03:38:38,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-25T03:38:38,378 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:38:38,378 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:38:38,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-25T03:38:38,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-25T03:38:38,380 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:38:38,381 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:38:38,381 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:38:38,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:38,385 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:38:38,385 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 629be9b3b266d56b676cd5a9595f19cd 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-25T03:38:38,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112550455c003b724638bee16168fba916e3_629be9b3b266d56b676cd5a9595f19cd is 50, key is test_row_0/A:col10/1732505916455/Put/seqid=0 2024-11-25T03:38:38,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742192_1368 (size=14794) 2024-11-25T03:38:38,398 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:38,401 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112550455c003b724638bee16168fba916e3_629be9b3b266d56b676cd5a9595f19cd to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112550455c003b724638bee16168fba916e3_629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:38,402 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/70404084fe194f33af92a32095e7ccd0, store: [table=TestAcidGuarantees family=A region=629be9b3b266d56b676cd5a9595f19cd] 2024-11-25T03:38:38,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/70404084fe194f33af92a32095e7ccd0 is 175, key is test_row_0/A:col10/1732505916455/Put/seqid=0 2024-11-25T03:38:38,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742193_1369 (size=39749) 2024-11-25T03:38:38,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:38,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36696 deadline: 1732505978434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:38,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:38,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36760 deadline: 1732505978435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:38,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:38,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:38,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:38,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:38,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:38,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:38,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-25T03:38:38,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-25T03:38:38,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:38,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. as already flushing 2024-11-25T03:38:38,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:38,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:38,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:38,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:38,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:38,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36696 deadline: 1732505978558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:38,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:38,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:38,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:38,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36760 deadline: 1732505978558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:38,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:38,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:38,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:38,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:38,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:38,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505978648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:38,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:38,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505978648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:38,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:38,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505978648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:38,658 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,34881,1732505806588 for 'test_row_1' in 629be9b3b266d56b676cd5a9595f19cd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:38:38,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:38,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:38,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:38,658 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,34881,1732505806588 for 'test_row_0' in 629be9b3b266d56b676cd5a9595f19cd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:38:38,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:38,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:38,658 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,34881,1732505806588 for 'test_row_0' in 629be9b3b266d56b676cd5a9595f19cd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:38:38,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:38,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:38,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:38,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:38,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-25T03:38:38,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:38,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. as already flushing 2024-11-25T03:38:38,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:38,686 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:38,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:38,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:38,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-25T03:38:38,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:38,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36696 deadline: 1732505978777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:38,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:38,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36760 deadline: 1732505978777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:38,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:38,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:38,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:38,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:38,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:38,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:38,808 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/70404084fe194f33af92a32095e7ccd0 2024-11-25T03:38:38,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/9d485cc543c94f31bde1a6a376e60578 is 50, key is test_row_0/B:col10/1732505916455/Put/seqid=0 2024-11-25T03:38:38,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742194_1370 (size=12151) 2024-11-25T03:38:38,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-25T03:38:38,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:38,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. as already flushing 2024-11-25T03:38:38,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:38,839 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:38,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:38,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:38,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-25T03:38:38,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:38,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. as already flushing 2024-11-25T03:38:38,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:38,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:38,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:38,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:39,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-25T03:38:39,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:39,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36696 deadline: 1732505979098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:39,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:39,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:39,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:39,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:39,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36760 deadline: 1732505979098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:39,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:39,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:39,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:39,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-25T03:38:39,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:39,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. as already flushing 2024-11-25T03:38:39,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:39,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:39,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:39,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:39,219 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/9d485cc543c94f31bde1a6a376e60578 2024-11-25T03:38:39,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/720dade7194d439c9b76d19585997714 is 50, key is test_row_0/C:col10/1732505916455/Put/seqid=0 2024-11-25T03:38:39,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742195_1371 (size=12151) 2024-11-25T03:38:39,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-25T03:38:39,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:39,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. as already flushing 2024-11-25T03:38:39,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:39,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:39,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:39,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:39,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-25T03:38:39,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:39,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. as already flushing 2024-11-25T03:38:39,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:39,451 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:39,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:39,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:39,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-25T03:38:39,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-25T03:38:39,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:39,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. as already flushing 2024-11-25T03:38:39,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:39,604 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:39,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:39,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:39,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:39,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36696 deadline: 1732505979617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:39,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:39,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36760 deadline: 1732505979618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:39,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:39,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:39,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:39,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:39,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:39,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:39,633 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/720dade7194d439c9b76d19585997714 2024-11-25T03:38:39,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/70404084fe194f33af92a32095e7ccd0 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/70404084fe194f33af92a32095e7ccd0 2024-11-25T03:38:39,641 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/70404084fe194f33af92a32095e7ccd0, entries=200, sequenceid=195, filesize=38.8 K 2024-11-25T03:38:39,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/9d485cc543c94f31bde1a6a376e60578 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/9d485cc543c94f31bde1a6a376e60578 2024-11-25T03:38:39,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/9d485cc543c94f31bde1a6a376e60578, entries=150, sequenceid=195, filesize=11.9 K 2024-11-25T03:38:39,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/720dade7194d439c9b76d19585997714 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/720dade7194d439c9b76d19585997714 2024-11-25T03:38:39,650 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/720dade7194d439c9b76d19585997714, entries=150, sequenceid=195, filesize=11.9 K 2024-11-25T03:38:39,651 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 629be9b3b266d56b676cd5a9595f19cd in 1266ms, sequenceid=195, compaction requested=true 2024-11-25T03:38:39,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 629be9b3b266d56b676cd5a9595f19cd: 2024-11-25T03:38:39,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 629be9b3b266d56b676cd5a9595f19cd:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:38:39,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:38:39,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 629be9b3b266d56b676cd5a9595f19cd:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:38:39,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:38:39,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 629be9b3b266d56b676cd5a9595f19cd:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:38:39,651 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:38:39,651 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:38:39,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:38:39,652 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:38:39,652 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133322 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:38:39,652 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 629be9b3b266d56b676cd5a9595f19cd/B is initiating minor compaction (all files) 2024-11-25T03:38:39,652 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): 629be9b3b266d56b676cd5a9595f19cd/A is initiating minor compaction (all files) 2024-11-25T03:38:39,653 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 629be9b3b266d56b676cd5a9595f19cd/A in TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:39,653 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 629be9b3b266d56b676cd5a9595f19cd/B in TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:39,653 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/e74f589ffac74e0cbb97a94e457a545a, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/16e6cd22eadb4f9a90c49b3a9dafb837, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/2950c32403e84d91b55aa3c9708544d7, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/70404084fe194f33af92a32095e7ccd0] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp, totalSize=130.2 K 2024-11-25T03:38:39,653 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/d0aa8f6cc3e84f70b55c0c1991844808, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/9e24bab4733c4ff4a3e0cb2ae0c959d4, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/249a0a66cde5448aa5d9cd286462955e, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/9d485cc543c94f31bde1a6a376e60578] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp, totalSize=47.7 K 2024-11-25T03:38:39,653 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:39,653 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. files: [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/e74f589ffac74e0cbb97a94e457a545a, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/16e6cd22eadb4f9a90c49b3a9dafb837, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/2950c32403e84d91b55aa3c9708544d7, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/70404084fe194f33af92a32095e7ccd0] 2024-11-25T03:38:39,653 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting e74f589ffac74e0cbb97a94e457a545a, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732505912322 2024-11-25T03:38:39,653 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting d0aa8f6cc3e84f70b55c0c1991844808, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732505912322 2024-11-25T03:38:39,654 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e24bab4733c4ff4a3e0cb2ae0c959d4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732505913538 2024-11-25T03:38:39,654 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16e6cd22eadb4f9a90c49b3a9dafb837, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732505913538 2024-11-25T03:38:39,654 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 249a0a66cde5448aa5d9cd286462955e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732505915747 2024-11-25T03:38:39,654 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2950c32403e84d91b55aa3c9708544d7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732505915747 2024-11-25T03:38:39,656 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70404084fe194f33af92a32095e7ccd0, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732505916455 2024-11-25T03:38:39,656 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d485cc543c94f31bde1a6a376e60578, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732505916455 2024-11-25T03:38:39,663 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=629be9b3b266d56b676cd5a9595f19cd] 2024-11-25T03:38:39,665 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 629be9b3b266d56b676cd5a9595f19cd#B#compaction#311 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:38:39,665 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/84bdcacac63e4018a50c3917b91ec4a3 is 50, key is test_row_0/B:col10/1732505916455/Put/seqid=0 2024-11-25T03:38:39,669 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125d17e6e11338d4072a3a0664b35325c73_629be9b3b266d56b676cd5a9595f19cd store=[table=TestAcidGuarantees family=A region=629be9b3b266d56b676cd5a9595f19cd] 2024-11-25T03:38:39,671 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125d17e6e11338d4072a3a0664b35325c73_629be9b3b266d56b676cd5a9595f19cd, store=[table=TestAcidGuarantees family=A region=629be9b3b266d56b676cd5a9595f19cd] 2024-11-25T03:38:39,671 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125d17e6e11338d4072a3a0664b35325c73_629be9b3b266d56b676cd5a9595f19cd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=629be9b3b266d56b676cd5a9595f19cd] 2024-11-25T03:38:39,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742196_1372 (size=12595) 2024-11-25T03:38:39,681 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/84bdcacac63e4018a50c3917b91ec4a3 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/84bdcacac63e4018a50c3917b91ec4a3 2024-11-25T03:38:39,688 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 629be9b3b266d56b676cd5a9595f19cd/B of 629be9b3b266d56b676cd5a9595f19cd into 84bdcacac63e4018a50c3917b91ec4a3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:38:39,688 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 629be9b3b266d56b676cd5a9595f19cd: 2024-11-25T03:38:39,688 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., storeName=629be9b3b266d56b676cd5a9595f19cd/B, priority=12, startTime=1732505919651; duration=0sec 2024-11-25T03:38:39,689 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:38:39,689 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 629be9b3b266d56b676cd5a9595f19cd:B 2024-11-25T03:38:39,689 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:38:39,690 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:38:39,690 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 629be9b3b266d56b676cd5a9595f19cd/C is initiating minor compaction (all files) 2024-11-25T03:38:39,690 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 629be9b3b266d56b676cd5a9595f19cd/C in TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:39,690 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/ebdfcf1e175f4987832e16dc4fea20ff, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/f6ab1f10e69e4b1e9f9d4e7e208c3973, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/d1ce8015cee242f1b84f2757c258d85b, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/720dade7194d439c9b76d19585997714] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp, totalSize=47.7 K 2024-11-25T03:38:39,690 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting ebdfcf1e175f4987832e16dc4fea20ff, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732505912322 2024-11-25T03:38:39,690 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting f6ab1f10e69e4b1e9f9d4e7e208c3973, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732505913538 2024-11-25T03:38:39,691 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting d1ce8015cee242f1b84f2757c258d85b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732505915747 2024-11-25T03:38:39,691 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 720dade7194d439c9b76d19585997714, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732505916455 2024-11-25T03:38:39,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742197_1373 (size=4469) 2024-11-25T03:38:39,701 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 629be9b3b266d56b676cd5a9595f19cd#C#compaction#313 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:38:39,702 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/9146c05379d24409950cb5a3a800ca6e is 50, key is test_row_0/C:col10/1732505916455/Put/seqid=0 2024-11-25T03:38:39,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742198_1374 (size=12595) 2024-11-25T03:38:39,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-25T03:38:39,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:39,757 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 629be9b3b266d56b676cd5a9595f19cd 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-25T03:38:39,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112505ac05cd01f2461e96531b929639ee38_629be9b3b266d56b676cd5a9595f19cd is 50, key is test_row_0/A:col10/1732505918432/Put/seqid=0 2024-11-25T03:38:39,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742199_1375 (size=12304) 2024-11-25T03:38:40,097 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 629be9b3b266d56b676cd5a9595f19cd#A#compaction#312 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:38:40,097 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/550fc08faf284f359409a0755beb87dd is 175, key is test_row_0/A:col10/1732505916455/Put/seqid=0 2024-11-25T03:38:40,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742200_1376 (size=31549) 2024-11-25T03:38:40,118 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/9146c05379d24409950cb5a3a800ca6e as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/9146c05379d24409950cb5a3a800ca6e 2024-11-25T03:38:40,121 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 629be9b3b266d56b676cd5a9595f19cd/C of 629be9b3b266d56b676cd5a9595f19cd into 9146c05379d24409950cb5a3a800ca6e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:38:40,121 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 629be9b3b266d56b676cd5a9595f19cd: 2024-11-25T03:38:40,121 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., storeName=629be9b3b266d56b676cd5a9595f19cd/C, priority=12, startTime=1732505919651; duration=0sec 2024-11-25T03:38:40,121 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:38:40,122 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 629be9b3b266d56b676cd5a9595f19cd:C 2024-11-25T03:38:40,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:40,173 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112505ac05cd01f2461e96531b929639ee38_629be9b3b266d56b676cd5a9595f19cd to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112505ac05cd01f2461e96531b929639ee38_629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:40,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/521ad50da73a421dbd46ee4665f2f22e, store: [table=TestAcidGuarantees family=A region=629be9b3b266d56b676cd5a9595f19cd] 2024-11-25T03:38:40,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/521ad50da73a421dbd46ee4665f2f22e is 175, key is test_row_0/A:col10/1732505918432/Put/seqid=0 2024-11-25T03:38:40,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742201_1377 (size=31105) 2024-11-25T03:38:40,179 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=209, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/521ad50da73a421dbd46ee4665f2f22e 2024-11-25T03:38:40,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/4f9139655c4743a2a58317a443240a46 is 50, key is test_row_0/B:col10/1732505918432/Put/seqid=0 2024-11-25T03:38:40,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742202_1378 (size=12151) 2024-11-25T03:38:40,507 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/550fc08faf284f359409a0755beb87dd as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/550fc08faf284f359409a0755beb87dd 2024-11-25T03:38:40,512 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 629be9b3b266d56b676cd5a9595f19cd/A of 629be9b3b266d56b676cd5a9595f19cd into 550fc08faf284f359409a0755beb87dd(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:38:40,512 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 629be9b3b266d56b676cd5a9595f19cd: 2024-11-25T03:38:40,512 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., storeName=629be9b3b266d56b676cd5a9595f19cd/A, priority=12, startTime=1732505919651; duration=0sec 2024-11-25T03:38:40,512 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:38:40,512 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 629be9b3b266d56b676cd5a9595f19cd:A 2024-11-25T03:38:40,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-25T03:38:40,593 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/4f9139655c4743a2a58317a443240a46 2024-11-25T03:38:40,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/591123d5e02a4e25acd417cce7f6aeeb is 50, key is test_row_0/C:col10/1732505918432/Put/seqid=0 2024-11-25T03:38:40,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742203_1379 (size=12151) 2024-11-25T03:38:40,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:40,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. as already flushing 2024-11-25T03:38:40,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:40,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36760 deadline: 1732505980703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:40,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:40,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:40,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:40,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505980704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:40,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:40,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:40,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:40,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:40,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:40,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505980706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:40,718 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:40,718 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:40,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:40,718 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:40,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505980706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:40,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:40,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:40,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:40,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:40,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36696 deadline: 1732505980707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:40,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:40,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:40,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:40,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:40,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36760 deadline: 1732505980817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:40,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:40,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:40,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505980817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:40,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:40,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:40,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:40,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:40,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:40,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:40,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505980828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:40,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:40,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505980828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:40,839 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:40,839 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:40,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:40,839 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:40,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36696 deadline: 1732505980828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:40,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:40,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:40,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:40,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:40,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:40,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:41,004 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/591123d5e02a4e25acd417cce7f6aeeb 2024-11-25T03:38:41,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/521ad50da73a421dbd46ee4665f2f22e as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/521ad50da73a421dbd46ee4665f2f22e 2024-11-25T03:38:41,012 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/521ad50da73a421dbd46ee4665f2f22e, entries=150, sequenceid=209, filesize=30.4 K 2024-11-25T03:38:41,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/4f9139655c4743a2a58317a443240a46 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/4f9139655c4743a2a58317a443240a46 2024-11-25T03:38:41,017 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/4f9139655c4743a2a58317a443240a46, entries=150, sequenceid=209, filesize=11.9 K 2024-11-25T03:38:41,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/591123d5e02a4e25acd417cce7f6aeeb as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/591123d5e02a4e25acd417cce7f6aeeb 2024-11-25T03:38:41,022 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/591123d5e02a4e25acd417cce7f6aeeb, entries=150, sequenceid=209, filesize=11.9 K 2024-11-25T03:38:41,022 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 629be9b3b266d56b676cd5a9595f19cd in 1265ms, sequenceid=209, compaction requested=false 2024-11-25T03:38:41,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 629be9b3b266d56b676cd5a9595f19cd: 2024-11-25T03:38:41,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:41,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-25T03:38:41,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-25T03:38:41,026 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-25T03:38:41,026 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6430 sec 2024-11-25T03:38:41,028 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.6490 sec 2024-11-25T03:38:41,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:41,041 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:38:41,041 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 629be9b3b266d56b676cd5a9595f19cd 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-25T03:38:41,049 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125025b4c5a0851480da92cb2aa33567dae_629be9b3b266d56b676cd5a9595f19cd is 50, key is test_row_0/A:col10/1732505921040/Put/seqid=0 2024-11-25T03:38:41,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742204_1380 (size=17284) 2024-11-25T03:38:41,055 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:41,060 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125025b4c5a0851480da92cb2aa33567dae_629be9b3b266d56b676cd5a9595f19cd to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125025b4c5a0851480da92cb2aa33567dae_629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:41,060 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/a26a68146aca4d9c805148e096821a40, store: [table=TestAcidGuarantees family=A region=629be9b3b266d56b676cd5a9595f19cd] 2024-11-25T03:38:41,061 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/a26a68146aca4d9c805148e096821a40 is 175, key is test_row_0/A:col10/1732505921040/Put/seqid=0 2024-11-25T03:38:41,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742205_1381 (size=48389) 2024-11-25T03:38:41,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:41,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36696 deadline: 1732505981058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:41,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:41,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:41,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:41,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:41,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36760 deadline: 1732505981059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:41,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:41,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:41,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:41,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:41,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505981065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:41,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:41,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:41,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:41,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:41,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505981072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:41,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:41,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:41,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505981072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:41,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:41,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:41,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:41,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:41,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:41,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:41,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36696 deadline: 1732505981177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:41,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:41,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36760 deadline: 1732505981177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:41,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:41,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:41,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:41,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:41,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:41,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:41,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:41,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505981177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:41,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:41,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:41,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:41,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:41,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505981187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:41,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:41,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:41,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505981187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:41,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:41,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:41,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:41,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:41,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:41,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:41,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36696 deadline: 1732505981387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:41,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:41,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:41,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:41,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36760 deadline: 1732505981387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:41,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:41,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:41,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:41,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:41,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:41,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505981387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:41,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:41,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:41,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:41,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:41,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505981397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:41,401 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:41,401 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:41,401 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:41,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:41,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505981397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:41,406 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:41,406 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:41,406 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:41,466 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=236, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/a26a68146aca4d9c805148e096821a40 2024-11-25T03:38:41,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/f42ea8ab90334ea485f0f550f0f07c1d is 50, key is test_row_0/B:col10/1732505921040/Put/seqid=0 2024-11-25T03:38:41,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742206_1382 (size=12151) 2024-11-25T03:38:41,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:41,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36696 deadline: 1732505981698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:41,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:41,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:41,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36760 deadline: 1732505981698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:41,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:41,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:41,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:41,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:41,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505981698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:41,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:41,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:41,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:41,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:41,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:41,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:41,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505981707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:41,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:41,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:41,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:41,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:41,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505981717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:41,721 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:41,721 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:41,721 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:41,877 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/f42ea8ab90334ea485f0f550f0f07c1d 2024-11-25T03:38:41,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/eca909f133f84d229964b5b7fe70fc55 is 50, key is test_row_0/C:col10/1732505921040/Put/seqid=0 2024-11-25T03:38:41,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742207_1383 (size=12151) 2024-11-25T03:38:42,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:42,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36696 deadline: 1732505982217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:42,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:42,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:42,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36760 deadline: 1732505982217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:42,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:42,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:42,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:42,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:42,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:42,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:42,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505982217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:42,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:42,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505982217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:42,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:42,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:42,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:42,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:42,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:42,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:42,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:42,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505982227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:42,237 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:42,237 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:42,237 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:42,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/eca909f133f84d229964b5b7fe70fc55 2024-11-25T03:38:42,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/a26a68146aca4d9c805148e096821a40 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/a26a68146aca4d9c805148e096821a40 2024-11-25T03:38:42,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/a26a68146aca4d9c805148e096821a40, entries=250, sequenceid=236, filesize=47.3 K 2024-11-25T03:38:42,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/f42ea8ab90334ea485f0f550f0f07c1d as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/f42ea8ab90334ea485f0f550f0f07c1d 2024-11-25T03:38:42,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/f42ea8ab90334ea485f0f550f0f07c1d, entries=150, sequenceid=236, filesize=11.9 K 2024-11-25T03:38:42,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/eca909f133f84d229964b5b7fe70fc55 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/eca909f133f84d229964b5b7fe70fc55 2024-11-25T03:38:42,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/eca909f133f84d229964b5b7fe70fc55, entries=150, sequenceid=236, filesize=11.9 K 2024-11-25T03:38:42,305 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 629be9b3b266d56b676cd5a9595f19cd in 1263ms, sequenceid=236, compaction requested=true 2024-11-25T03:38:42,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 629be9b3b266d56b676cd5a9595f19cd: 2024-11-25T03:38:42,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 629be9b3b266d56b676cd5a9595f19cd:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:38:42,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:38:42,305 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:38:42,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,305 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:38:42,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 629be9b3b266d56b676cd5a9595f19cd:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:38:42,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:38:42,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 629be9b3b266d56b676cd5a9595f19cd:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:38:42,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:38:42,306 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111043 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:38:42,306 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): 629be9b3b266d56b676cd5a9595f19cd/A is initiating minor compaction (all files) 2024-11-25T03:38:42,306 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:38:42,306 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 629be9b3b266d56b676cd5a9595f19cd/A in TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:42,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,306 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 629be9b3b266d56b676cd5a9595f19cd/B is initiating minor compaction (all files) 2024-11-25T03:38:42,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,306 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/550fc08faf284f359409a0755beb87dd, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/521ad50da73a421dbd46ee4665f2f22e, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/a26a68146aca4d9c805148e096821a40] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp, totalSize=108.4 K 2024-11-25T03:38:42,306 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 629be9b3b266d56b676cd5a9595f19cd/B in TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:42,306 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:42,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,306 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. files: [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/550fc08faf284f359409a0755beb87dd, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/521ad50da73a421dbd46ee4665f2f22e, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/a26a68146aca4d9c805148e096821a40] 2024-11-25T03:38:42,306 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/84bdcacac63e4018a50c3917b91ec4a3, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/4f9139655c4743a2a58317a443240a46, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/f42ea8ab90334ea485f0f550f0f07c1d] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp, totalSize=36.0 K 2024-11-25T03:38:42,307 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 84bdcacac63e4018a50c3917b91ec4a3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732505916455 2024-11-25T03:38:42,307 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 550fc08faf284f359409a0755beb87dd, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732505916455 2024-11-25T03:38:42,307 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 521ad50da73a421dbd46ee4665f2f22e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732505918396 2024-11-25T03:38:42,307 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f9139655c4743a2a58317a443240a46, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732505918396 2024-11-25T03:38:42,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,307 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting a26a68146aca4d9c805148e096821a40, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732505920702 2024-11-25T03:38:42,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,308 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting f42ea8ab90334ea485f0f550f0f07c1d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732505920705 2024-11-25T03:38:42,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,314 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=629be9b3b266d56b676cd5a9595f19cd] 2024-11-25T03:38:42,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,317 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125fbdc78a01c7b4ff197562d15448ab9fd_629be9b3b266d56b676cd5a9595f19cd store=[table=TestAcidGuarantees family=A region=629be9b3b266d56b676cd5a9595f19cd] 2024-11-25T03:38:42,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,323 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125fbdc78a01c7b4ff197562d15448ab9fd_629be9b3b266d56b676cd5a9595f19cd, store=[table=TestAcidGuarantees family=A region=629be9b3b266d56b676cd5a9595f19cd] 2024-11-25T03:38:42,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,323 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125fbdc78a01c7b4ff197562d15448ab9fd_629be9b3b266d56b676cd5a9595f19cd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=629be9b3b266d56b676cd5a9595f19cd] 2024-11-25T03:38:42,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,326 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 629be9b3b266d56b676cd5a9595f19cd#B#compaction#321 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:38:42,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,327 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/810f9f0a939d455a844a664980f64156 is 50, key is test_row_0/B:col10/1732505921040/Put/seqid=0 2024-11-25T03:38:42,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742208_1384 (size=4469) 2024-11-25T03:38:42,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,343 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 629be9b3b266d56b676cd5a9595f19cd#A#compaction#320 average throughput is 0.84 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:38:42,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,343 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/e6894af09e924016a39f9a996599f9f5 is 175, key is test_row_0/A:col10/1732505921040/Put/seqid=0 2024-11-25T03:38:42,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742209_1385 (size=12697) 2024-11-25T03:38:42,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742210_1386 (size=31651) 2024-11-25T03:38:42,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,378 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/e6894af09e924016a39f9a996599f9f5 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/e6894af09e924016a39f9a996599f9f5 2024-11-25T03:38:42,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,381 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/810f9f0a939d455a844a664980f64156 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/810f9f0a939d455a844a664980f64156 2024-11-25T03:38:42,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,384 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 629be9b3b266d56b676cd5a9595f19cd/A of 629be9b3b266d56b676cd5a9595f19cd into e6894af09e924016a39f9a996599f9f5(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:38:42,384 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 629be9b3b266d56b676cd5a9595f19cd: 2024-11-25T03:38:42,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,384 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., storeName=629be9b3b266d56b676cd5a9595f19cd/A, priority=13, startTime=1732505922305; duration=0sec 2024-11-25T03:38:42,385 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:38:42,385 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 629be9b3b266d56b676cd5a9595f19cd:A 2024-11-25T03:38:42,385 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:38:42,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,386 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:38:42,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,386 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): 629be9b3b266d56b676cd5a9595f19cd/C is initiating minor compaction (all files) 2024-11-25T03:38:42,386 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 629be9b3b266d56b676cd5a9595f19cd/C in TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:42,386 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/9146c05379d24409950cb5a3a800ca6e, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/591123d5e02a4e25acd417cce7f6aeeb, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/eca909f133f84d229964b5b7fe70fc55] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp, totalSize=36.0 K 2024-11-25T03:38:42,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,386 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9146c05379d24409950cb5a3a800ca6e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732505916455 2024-11-25T03:38:42,387 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 591123d5e02a4e25acd417cce7f6aeeb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732505918396 2024-11-25T03:38:42,387 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting eca909f133f84d229964b5b7fe70fc55, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732505920705 2024-11-25T03:38:42,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,387 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 629be9b3b266d56b676cd5a9595f19cd/B of 629be9b3b266d56b676cd5a9595f19cd into 810f9f0a939d455a844a664980f64156(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:38:42,387 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 629be9b3b266d56b676cd5a9595f19cd: 2024-11-25T03:38:42,388 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., storeName=629be9b3b266d56b676cd5a9595f19cd/B, priority=13, startTime=1732505922305; duration=0sec 2024-11-25T03:38:42,388 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:38:42,388 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 629be9b3b266d56b676cd5a9595f19cd:B 2024-11-25T03:38:42,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,395 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 629be9b3b266d56b676cd5a9595f19cd#C#compaction#322 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:38:42,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,395 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/9fe8b6c9f04744a980106beda0b09ac3 is 50, key is test_row_0/C:col10/1732505921040/Put/seqid=0 2024-11-25T03:38:42,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742211_1387 (size=12697) 2024-11-25T03:38:42,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-25T03:38:42,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,568 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:38:42,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,568 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:38:42,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-25T03:38:42,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,571 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:38:42,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-25T03:38:42,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,572 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:38:42,572 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:38:42,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-25T03:38:42,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-25T03:38:42,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:42,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,725 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 629be9b3b266d56b676cd5a9595f19cd 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-25T03:38:42,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125d12ff2930cf34884bcbe8d53aa4cb6cb_629be9b3b266d56b676cd5a9595f19cd is 50, key is test_row_0/A:col10/1732505921063/Put/seqid=0 2024-11-25T03:38:42,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742212_1388 (size=9814) 2024-11-25T03:38:42,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,812 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/9fe8b6c9f04744a980106beda0b09ac3 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/9fe8b6c9f04744a980106beda0b09ac3 2024-11-25T03:38:42,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,817 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 629be9b3b266d56b676cd5a9595f19cd/C of 629be9b3b266d56b676cd5a9595f19cd into 9fe8b6c9f04744a980106beda0b09ac3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:38:42,817 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 629be9b3b266d56b676cd5a9595f19cd: 2024-11-25T03:38:42,817 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., storeName=629be9b3b266d56b676cd5a9595f19cd/C, priority=13, startTime=1732505922305; duration=0sec 2024-11-25T03:38:42,817 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:38:42,817 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 629be9b3b266d56b676cd5a9595f19cd:C 2024-11-25T03:38:42,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-25T03:38:42,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:42,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,156 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125d12ff2930cf34884bcbe8d53aa4cb6cb_629be9b3b266d56b676cd5a9595f19cd to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125d12ff2930cf34884bcbe8d53aa4cb6cb_629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:43,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/9664eff11a7d4a2a958fc09cb3fd0e42, store: [table=TestAcidGuarantees family=A region=629be9b3b266d56b676cd5a9595f19cd] 2024-11-25T03:38:43,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/9664eff11a7d4a2a958fc09cb3fd0e42 is 175, key is test_row_0/A:col10/1732505921063/Put/seqid=0 2024-11-25T03:38:43,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742213_1389 (size=22461) 2024-11-25T03:38:43,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-25T03:38:43,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:43,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. as already flushing 2024-11-25T03:38:43,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:43,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:43,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36760 deadline: 1732505983351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:43,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:43,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:43,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505983351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:43,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:43,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:43,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:43,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:43,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505983351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:43,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:43,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:43,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:43,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:43,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505983353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:43,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:43,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:43,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:43,366 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:43,366 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:43,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36696 deadline: 1732505983354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:43,366 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:43,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:43,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:43,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:43,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:43,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36760 deadline: 1732505983467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:43,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:43,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:43,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:43,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:43,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505983478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:43,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:43,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:43,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505983478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:43,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:43,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:43,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:43,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:43,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:43,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:43,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505983478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:43,483 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:43,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:43,483 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:43,483 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:43,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36696 deadline: 1732505983478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:43,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:43,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:43,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:43,571 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=250, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/9664eff11a7d4a2a958fc09cb3fd0e42 2024-11-25T03:38:43,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/2cdf4312a5ef415dab0b3ac0f561388e is 50, key is test_row_0/B:col10/1732505921063/Put/seqid=0 2024-11-25T03:38:43,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742214_1390 (size=9757) 2024-11-25T03:38:43,586 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/2cdf4312a5ef415dab0b3ac0f561388e 2024-11-25T03:38:43,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/c3b80897663e472da4f1cb08dee19664 is 50, key is test_row_0/C:col10/1732505921063/Put/seqid=0 2024-11-25T03:38:43,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742215_1391 (size=9757) 2024-11-25T03:38:43,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:43,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36760 deadline: 1732505983677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:43,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:43,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:43,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:43,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:43,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505983688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:43,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:43,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505983688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:43,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:43,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:43,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:43,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:43,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:43,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:43,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505983688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:43,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:43,696 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:43,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:43,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:43,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:43,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36696 deadline: 1732505983688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:43,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:43,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:43,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:43,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-25T03:38:43,996 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/c3b80897663e472da4f1cb08dee19664 2024-11-25T03:38:44,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/9664eff11a7d4a2a958fc09cb3fd0e42 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/9664eff11a7d4a2a958fc09cb3fd0e42 2024-11-25T03:38:44,004 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/9664eff11a7d4a2a958fc09cb3fd0e42, entries=100, sequenceid=250, filesize=21.9 K 2024-11-25T03:38:44,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:44,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36760 deadline: 1732505983998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:44,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:44,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:44,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:44,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/2cdf4312a5ef415dab0b3ac0f561388e as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/2cdf4312a5ef415dab0b3ac0f561388e 2024-11-25T03:38:44,009 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/2cdf4312a5ef415dab0b3ac0f561388e, entries=100, sequenceid=250, filesize=9.5 K 2024-11-25T03:38:44,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/c3b80897663e472da4f1cb08dee19664 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/c3b80897663e472da4f1cb08dee19664 2024-11-25T03:38:44,013 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/c3b80897663e472da4f1cb08dee19664, entries=100, sequenceid=250, filesize=9.5 K 2024-11-25T03:38:44,014 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 629be9b3b266d56b676cd5a9595f19cd in 1289ms, sequenceid=250, compaction requested=false 2024-11-25T03:38:44,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 629be9b3b266d56b676cd5a9595f19cd: 2024-11-25T03:38:44,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:44,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-25T03:38:44,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-25T03:38:44,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:44,017 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:38:44,017 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 629be9b3b266d56b676cd5a9595f19cd 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-25T03:38:44,018 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-25T03:38:44,018 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4440 sec 2024-11-25T03:38:44,020 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.4500 sec 2024-11-25T03:38:44,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112552cdfd0c4ca743e999dbe58c67288bda_629be9b3b266d56b676cd5a9595f19cd is 50, key is test_row_0/A:col10/1732505923315/Put/seqid=0 2024-11-25T03:38:44,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:44,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36696 deadline: 1732505984029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:44,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:44,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:44,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:44,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505984030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:44,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:44,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:44,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:44,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:44,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742216_1392 (size=14994) 2024-11-25T03:38:44,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:44,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505984031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:44,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:44,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505984031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:44,041 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:44,041 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:44,041 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:44,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:44,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:44,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:44,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:44,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36696 deadline: 1732505984137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:44,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:44,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:44,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:44,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505984137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:44,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:44,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:44,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:44,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:44,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:44,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505984147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:44,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:44,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:44,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:44,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505984148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:44,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:44,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:44,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:44,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:44,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:44,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36696 deadline: 1732505984347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:44,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:44,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:44,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:44,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505984347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:44,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:44,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:44,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:44,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:44,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:44,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505984357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:44,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:44,363 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:44,363 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:44,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505984357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:44,363 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:44,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:44,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:44,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:44,438 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:44,441 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112552cdfd0c4ca743e999dbe58c67288bda_629be9b3b266d56b676cd5a9595f19cd to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112552cdfd0c4ca743e999dbe58c67288bda_629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:44,442 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/9991639f666642589eb07e14a0042365, store: [table=TestAcidGuarantees family=A region=629be9b3b266d56b676cd5a9595f19cd] 2024-11-25T03:38:44,443 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/9991639f666642589eb07e14a0042365 is 175, key is test_row_0/A:col10/1732505923315/Put/seqid=0 2024-11-25T03:38:44,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742217_1393 (size=39949) 2024-11-25T03:38:44,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:44,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36760 deadline: 1732505984518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:44,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:44,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:44,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:44,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:44,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505984658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:44,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:44,666 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:44,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36696 deadline: 1732505984658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:44,666 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:44,666 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:44,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:44,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:44,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:44,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:44,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505984667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:44,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:44,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:44,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505984667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:44,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:44,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:44,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:44,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:44,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:44,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-25T03:38:44,718 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:38:44,718 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:38:44,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-25T03:38:44,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-25T03:38:44,720 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:38:44,721 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:38:44,721 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:38:44,799 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T03:38:44,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-25T03:38:44,848 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=277, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/9991639f666642589eb07e14a0042365 2024-11-25T03:38:44,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/defeaf1adc4c4f7b9325b1c040e34587 is 50, key is test_row_0/B:col10/1732505923315/Put/seqid=0 2024-11-25T03:38:44,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742218_1394 (size=12301) 2024-11-25T03:38:44,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-25T03:38:44,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:44,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. as already flushing 2024-11-25T03:38:44,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:44,873 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:44,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:44,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:45,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-25T03:38:45,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:45,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. as already flushing 2024-11-25T03:38:45,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:45,026 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:45,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:45,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:45,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-25T03:38:45,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-25T03:38:45,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:45,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. as already flushing 2024-11-25T03:38:45,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:45,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:45,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:45,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:45,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:45,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36772 deadline: 1732505985177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:45,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:45,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:45,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36696 deadline: 1732505985177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:45,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:45,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:45,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:45,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:45,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:45,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:45,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36788 deadline: 1732505985187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:45,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:45,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:45,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:45,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:45,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36712 deadline: 1732505985187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:45,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:45,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:45,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:45,270 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/defeaf1adc4c4f7b9325b1c040e34587 2024-11-25T03:38:45,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/60a1e35c3f724a39baf26de9a5046342 is 50, key is test_row_0/C:col10/1732505923315/Put/seqid=0 2024-11-25T03:38:45,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742219_1395 (size=12301) 2024-11-25T03:38:45,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-25T03:38:45,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:45,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. as already flushing 2024-11-25T03:38:45,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:45,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:45,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:45,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:45,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-25T03:38:45,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-25T03:38:45,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:45,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. as already flushing 2024-11-25T03:38:45,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:45,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:45,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:45,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:45,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:45,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36760 deadline: 1732505985537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:45,541 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,34881,1732505806588 for 'test_row_2' in 629be9b3b266d56b676cd5a9595f19cd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:38:45,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:45,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:45,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:38:45,637 INFO [Thread-1483 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1483. 2024-11-25T03:38:45,637 DEBUG [Thread-1483 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:38:45,637 DEBUG [Thread-1483 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:45,637 DEBUG [Thread-1483 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:45,637 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:38:45,637 INFO [Thread-1485 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1485. 2024-11-25T03:38:45,638 DEBUG [Thread-1485 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:38:45,638 DEBUG [Thread-1485 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:45,638 DEBUG [Thread-1485 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:45,638 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:38:45,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-25T03:38:45,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:45,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. as already flushing 2024-11-25T03:38:45,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:45,639 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:45,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:45,639 INFO [Thread-1479 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1479. 2024-11-25T03:38:45,639 DEBUG [Thread-1479 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:38:45,639 DEBUG [Thread-1479 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:45,640 DEBUG [Thread-1479 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:45,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:45,640 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:38:45,646 INFO [Thread-1481 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1481. 2024-11-25T03:38:45,646 DEBUG [Thread-1481 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:38:45,646 DEBUG [Thread-1481 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:45,646 DEBUG [Thread-1481 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:45,646 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:38:45,648 INFO [Thread-1487 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1487. 2024-11-25T03:38:45,648 DEBUG [Thread-1487 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:38:45,648 DEBUG [Thread-1487 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:45,648 DEBUG [Thread-1487 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:45,648 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:38:45,680 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/60a1e35c3f724a39baf26de9a5046342 2024-11-25T03:38:45,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/9991639f666642589eb07e14a0042365 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/9991639f666642589eb07e14a0042365 2024-11-25T03:38:45,687 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/9991639f666642589eb07e14a0042365, entries=200, sequenceid=277, filesize=39.0 K 2024-11-25T03:38:45,688 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/defeaf1adc4c4f7b9325b1c040e34587 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/defeaf1adc4c4f7b9325b1c040e34587 2024-11-25T03:38:45,690 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/defeaf1adc4c4f7b9325b1c040e34587, entries=150, sequenceid=277, filesize=12.0 K 2024-11-25T03:38:45,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/60a1e35c3f724a39baf26de9a5046342 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/60a1e35c3f724a39baf26de9a5046342 2024-11-25T03:38:45,693 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/60a1e35c3f724a39baf26de9a5046342, entries=150, sequenceid=277, filesize=12.0 K 2024-11-25T03:38:45,694 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 629be9b3b266d56b676cd5a9595f19cd in 1677ms, sequenceid=277, compaction requested=true 2024-11-25T03:38:45,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 629be9b3b266d56b676cd5a9595f19cd: 2024-11-25T03:38:45,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 629be9b3b266d56b676cd5a9595f19cd:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:38:45,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:38:45,694 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:38:45,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 629be9b3b266d56b676cd5a9595f19cd:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:38:45,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:38:45,694 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:38:45,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 629be9b3b266d56b676cd5a9595f19cd:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:38:45,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:38:45,695 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:38:45,695 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:38:45,695 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): 629be9b3b266d56b676cd5a9595f19cd/A is initiating minor compaction (all files) 2024-11-25T03:38:45,695 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 629be9b3b266d56b676cd5a9595f19cd/B is initiating minor compaction (all files) 2024-11-25T03:38:45,695 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 629be9b3b266d56b676cd5a9595f19cd/A in TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:45,695 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 629be9b3b266d56b676cd5a9595f19cd/B in TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:45,695 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/810f9f0a939d455a844a664980f64156, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/2cdf4312a5ef415dab0b3ac0f561388e, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/defeaf1adc4c4f7b9325b1c040e34587] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp, totalSize=33.9 K 2024-11-25T03:38:45,695 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/e6894af09e924016a39f9a996599f9f5, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/9664eff11a7d4a2a958fc09cb3fd0e42, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/9991639f666642589eb07e14a0042365] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp, totalSize=91.9 K 2024-11-25T03:38:45,695 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:45,695 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. files: [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/e6894af09e924016a39f9a996599f9f5, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/9664eff11a7d4a2a958fc09cb3fd0e42, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/9991639f666642589eb07e14a0042365] 2024-11-25T03:38:45,696 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 810f9f0a939d455a844a664980f64156, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732505920705 2024-11-25T03:38:45,696 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6894af09e924016a39f9a996599f9f5, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732505920705 2024-11-25T03:38:45,696 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cdf4312a5ef415dab0b3ac0f561388e, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732505921063 2024-11-25T03:38:45,696 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9664eff11a7d4a2a958fc09cb3fd0e42, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732505921063 2024-11-25T03:38:45,696 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting defeaf1adc4c4f7b9325b1c040e34587, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732505923315 2024-11-25T03:38:45,696 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9991639f666642589eb07e14a0042365, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732505923315 2024-11-25T03:38:45,701 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=629be9b3b266d56b676cd5a9595f19cd] 2024-11-25T03:38:45,702 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 629be9b3b266d56b676cd5a9595f19cd#B#compaction#329 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:38:45,702 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/3759326503e24290b13c7dae39bb67e5 is 50, key is test_row_0/B:col10/1732505923315/Put/seqid=0 2024-11-25T03:38:45,702 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125f29e10176ffc4086bed7275a2000d8c5_629be9b3b266d56b676cd5a9595f19cd store=[table=TestAcidGuarantees family=A region=629be9b3b266d56b676cd5a9595f19cd] 2024-11-25T03:38:45,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742220_1396 (size=12949) 2024-11-25T03:38:45,705 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125f29e10176ffc4086bed7275a2000d8c5_629be9b3b266d56b676cd5a9595f19cd, store=[table=TestAcidGuarantees family=A region=629be9b3b266d56b676cd5a9595f19cd] 2024-11-25T03:38:45,705 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125f29e10176ffc4086bed7275a2000d8c5_629be9b3b266d56b676cd5a9595f19cd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=629be9b3b266d56b676cd5a9595f19cd] 2024-11-25T03:38:45,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742221_1397 (size=4469) 2024-11-25T03:38:45,709 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/3759326503e24290b13c7dae39bb67e5 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/3759326503e24290b13c7dae39bb67e5 2024-11-25T03:38:45,712 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 629be9b3b266d56b676cd5a9595f19cd/B of 629be9b3b266d56b676cd5a9595f19cd into 3759326503e24290b13c7dae39bb67e5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:38:45,712 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 629be9b3b266d56b676cd5a9595f19cd: 2024-11-25T03:38:45,712 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., storeName=629be9b3b266d56b676cd5a9595f19cd/B, priority=13, startTime=1732505925694; duration=0sec 2024-11-25T03:38:45,713 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:38:45,713 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 629be9b3b266d56b676cd5a9595f19cd:B 2024-11-25T03:38:45,713 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:38:45,713 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:38:45,713 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 629be9b3b266d56b676cd5a9595f19cd/C is initiating minor compaction (all files) 2024-11-25T03:38:45,713 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 629be9b3b266d56b676cd5a9595f19cd/C in TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:45,713 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/9fe8b6c9f04744a980106beda0b09ac3, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/c3b80897663e472da4f1cb08dee19664, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/60a1e35c3f724a39baf26de9a5046342] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp, totalSize=33.9 K 2024-11-25T03:38:45,714 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fe8b6c9f04744a980106beda0b09ac3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732505920705 2024-11-25T03:38:45,714 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting c3b80897663e472da4f1cb08dee19664, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732505921063 2024-11-25T03:38:45,714 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 60a1e35c3f724a39baf26de9a5046342, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732505923315 2024-11-25T03:38:45,720 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 629be9b3b266d56b676cd5a9595f19cd#C#compaction#331 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:38:45,721 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/e94e5d943cde4a9a9b9ef3bf4cbcc828 is 50, key is test_row_0/C:col10/1732505923315/Put/seqid=0 2024-11-25T03:38:45,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742222_1398 (size=12949) 2024-11-25T03:38:45,792 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-25T03:38:45,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:45,792 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 629be9b3b266d56b676cd5a9595f19cd 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-25T03:38:45,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411256aa8592636364b8a96a58910f4341129_629be9b3b266d56b676cd5a9595f19cd is 50, key is test_row_0/A:col10/1732505924024/Put/seqid=0 2024-11-25T03:38:45,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742223_1399 (size=12454) 2024-11-25T03:38:45,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-25T03:38:46,110 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 629be9b3b266d56b676cd5a9595f19cd#A#compaction#330 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:38:46,111 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/f3c34f75bb9a4fec9f79c7305a8d82b3 is 175, key is test_row_0/A:col10/1732505923315/Put/seqid=0 2024-11-25T03:38:46,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742224_1400 (size=31903) 2024-11-25T03:38:46,128 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/e94e5d943cde4a9a9b9ef3bf4cbcc828 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/e94e5d943cde4a9a9b9ef3bf4cbcc828 2024-11-25T03:38:46,132 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 629be9b3b266d56b676cd5a9595f19cd/C of 629be9b3b266d56b676cd5a9595f19cd into e94e5d943cde4a9a9b9ef3bf4cbcc828(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:38:46,133 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 629be9b3b266d56b676cd5a9595f19cd: 2024-11-25T03:38:46,133 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., storeName=629be9b3b266d56b676cd5a9595f19cd/C, priority=13, startTime=1732505925694; duration=0sec 2024-11-25T03:38:46,133 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:38:46,133 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 629be9b3b266d56b676cd5a9595f19cd:C 2024-11-25T03:38:46,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:46,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. as already flushing 2024-11-25T03:38:46,199 INFO [Thread-1470 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1470. 2024-11-25T03:38:46,199 DEBUG [Thread-1470 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:38:46,199 DEBUG [Thread-1470 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:46,199 INFO [Thread-1474 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1474. 2024-11-25T03:38:46,200 DEBUG [Thread-1474 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:38:46,200 DEBUG [Thread-1474 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:46,200 DEBUG [Thread-1470 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:46,200 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:38:46,200 DEBUG [Thread-1474 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:46,200 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:38:46,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:46,206 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411256aa8592636364b8a96a58910f4341129_629be9b3b266d56b676cd5a9595f19cd to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411256aa8592636364b8a96a58910f4341129_629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:46,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/884fdc82575941a08467f052d4a6ca2c, store: [table=TestAcidGuarantees family=A region=629be9b3b266d56b676cd5a9595f19cd] 2024-11-25T03:38:46,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/884fdc82575941a08467f052d4a6ca2c is 175, key is test_row_0/A:col10/1732505924024/Put/seqid=0 2024-11-25T03:38:46,209 INFO [Thread-1468 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1468. 2024-11-25T03:38:46,209 INFO [Thread-1476 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1476. 2024-11-25T03:38:46,209 DEBUG [Thread-1468 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:38:46,209 DEBUG [Thread-1468 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:46,209 DEBUG [Thread-1476 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:38:46,209 DEBUG [Thread-1476 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:46,209 DEBUG [Thread-1468 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:46,209 DEBUG [Thread-1476 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:46,209 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:38:46,209 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:38:46,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742225_1401 (size=31255) 2024-11-25T03:38:46,519 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/f3c34f75bb9a4fec9f79c7305a8d82b3 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/f3c34f75bb9a4fec9f79c7305a8d82b3 2024-11-25T03:38:46,522 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 629be9b3b266d56b676cd5a9595f19cd/A of 629be9b3b266d56b676cd5a9595f19cd into f3c34f75bb9a4fec9f79c7305a8d82b3(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:38:46,523 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 629be9b3b266d56b676cd5a9595f19cd: 2024-11-25T03:38:46,523 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., storeName=629be9b3b266d56b676cd5a9595f19cd/A, priority=13, startTime=1732505925694; duration=0sec 2024-11-25T03:38:46,523 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:38:46,523 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 629be9b3b266d56b676cd5a9595f19cd:A 2024-11-25T03:38:46,612 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=288, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/884fdc82575941a08467f052d4a6ca2c 2024-11-25T03:38:46,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/557813bc507a47d3952e7a617a6309b8 is 50, key is test_row_0/B:col10/1732505924024/Put/seqid=0 2024-11-25T03:38:46,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742226_1402 (size=12301) 2024-11-25T03:38:46,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-25T03:38:47,021 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/557813bc507a47d3952e7a617a6309b8 2024-11-25T03:38:47,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/f8a8ee40bf1e4e1ab83f416d53afcce8 is 50, key is test_row_0/C:col10/1732505924024/Put/seqid=0 2024-11-25T03:38:47,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742227_1403 (size=12301) 2024-11-25T03:38:47,430 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/f8a8ee40bf1e4e1ab83f416d53afcce8 2024-11-25T03:38:47,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/884fdc82575941a08467f052d4a6ca2c as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/884fdc82575941a08467f052d4a6ca2c 2024-11-25T03:38:47,437 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/884fdc82575941a08467f052d4a6ca2c, entries=150, sequenceid=288, filesize=30.5 K 2024-11-25T03:38:47,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/557813bc507a47d3952e7a617a6309b8 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/557813bc507a47d3952e7a617a6309b8 2024-11-25T03:38:47,440 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/557813bc507a47d3952e7a617a6309b8, entries=150, sequenceid=288, filesize=12.0 K 2024-11-25T03:38:47,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/f8a8ee40bf1e4e1ab83f416d53afcce8 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/f8a8ee40bf1e4e1ab83f416d53afcce8 2024-11-25T03:38:47,443 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/f8a8ee40bf1e4e1ab83f416d53afcce8, entries=150, sequenceid=288, filesize=12.0 K 2024-11-25T03:38:47,443 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=26.84 KB/27480 for 629be9b3b266d56b676cd5a9595f19cd in 1651ms, sequenceid=288, compaction requested=false 2024-11-25T03:38:47,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 629be9b3b266d56b676cd5a9595f19cd: 2024-11-25T03:38:47,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:47,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-25T03:38:47,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-25T03:38:47,446 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-25T03:38:47,446 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7240 sec 2024-11-25T03:38:47,447 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.7280 sec 2024-11-25T03:38:47,568 INFO [Thread-1472 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1472. 2024-11-25T03:38:47,568 DEBUG [Thread-1472 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:38:47,568 DEBUG [Thread-1472 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:47,568 DEBUG [Thread-1472 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:47,568 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:38:48,751 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/12fecffaebcd49cc8dc9d9f003bc5f70, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/3772b5eaa7944a7394b50d74cf51964d, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/e59bc0c6c84d41028a65e8140df5d86c, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/814f1a81d07d49259070358a64ec96d9, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/ec2e9320a04a40f2a906b7addb7ee82e, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/389ad6a18aaa42ebbba86b7689ca017a, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/bd9dd7f21ba34c778b9c0ec6d4dd620f, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/12cb4e7b51e74f9da84cb44d6f911263, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/36813e12e0974dc09bb555701652cfd9, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/e74f589ffac74e0cbb97a94e457a545a, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/16e6cd22eadb4f9a90c49b3a9dafb837, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/2950c32403e84d91b55aa3c9708544d7, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/70404084fe194f33af92a32095e7ccd0, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/550fc08faf284f359409a0755beb87dd, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/521ad50da73a421dbd46ee4665f2f22e, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/a26a68146aca4d9c805148e096821a40, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/e6894af09e924016a39f9a996599f9f5, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/9664eff11a7d4a2a958fc09cb3fd0e42, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/9991639f666642589eb07e14a0042365] to archive 2024-11-25T03:38:48,752 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T03:38:48,753 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/12fecffaebcd49cc8dc9d9f003bc5f70 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/12fecffaebcd49cc8dc9d9f003bc5f70 2024-11-25T03:38:48,754 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/3772b5eaa7944a7394b50d74cf51964d to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/3772b5eaa7944a7394b50d74cf51964d 2024-11-25T03:38:48,755 WARN [regionserver/38a5d2077268:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 2, running: 1 2024-11-25T03:38:48,755 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/e59bc0c6c84d41028a65e8140df5d86c to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/e59bc0c6c84d41028a65e8140df5d86c 2024-11-25T03:38:48,756 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/814f1a81d07d49259070358a64ec96d9 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/814f1a81d07d49259070358a64ec96d9 2024-11-25T03:38:48,757 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/ec2e9320a04a40f2a906b7addb7ee82e to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/ec2e9320a04a40f2a906b7addb7ee82e 2024-11-25T03:38:48,757 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/389ad6a18aaa42ebbba86b7689ca017a to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/389ad6a18aaa42ebbba86b7689ca017a 2024-11-25T03:38:48,758 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/bd9dd7f21ba34c778b9c0ec6d4dd620f to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/bd9dd7f21ba34c778b9c0ec6d4dd620f 2024-11-25T03:38:48,759 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/12cb4e7b51e74f9da84cb44d6f911263 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/12cb4e7b51e74f9da84cb44d6f911263 2024-11-25T03:38:48,760 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/36813e12e0974dc09bb555701652cfd9 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/36813e12e0974dc09bb555701652cfd9 2024-11-25T03:38:48,760 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/e74f589ffac74e0cbb97a94e457a545a to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/e74f589ffac74e0cbb97a94e457a545a 2024-11-25T03:38:48,761 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/16e6cd22eadb4f9a90c49b3a9dafb837 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/16e6cd22eadb4f9a90c49b3a9dafb837 2024-11-25T03:38:48,762 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/2950c32403e84d91b55aa3c9708544d7 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/2950c32403e84d91b55aa3c9708544d7 2024-11-25T03:38:48,763 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/70404084fe194f33af92a32095e7ccd0 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/70404084fe194f33af92a32095e7ccd0 2024-11-25T03:38:48,763 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/550fc08faf284f359409a0755beb87dd to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/550fc08faf284f359409a0755beb87dd 2024-11-25T03:38:48,764 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/521ad50da73a421dbd46ee4665f2f22e to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/521ad50da73a421dbd46ee4665f2f22e 2024-11-25T03:38:48,765 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/a26a68146aca4d9c805148e096821a40 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/a26a68146aca4d9c805148e096821a40 2024-11-25T03:38:48,766 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/e6894af09e924016a39f9a996599f9f5 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/e6894af09e924016a39f9a996599f9f5 2024-11-25T03:38:48,766 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/9664eff11a7d4a2a958fc09cb3fd0e42 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/9664eff11a7d4a2a958fc09cb3fd0e42 2024-11-25T03:38:48,767 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/9991639f666642589eb07e14a0042365 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/9991639f666642589eb07e14a0042365 2024-11-25T03:38:48,770 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/b474b691a1cb4e8883d8511a7ecdd6ca, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/5bddac04ab7f448f85f14f3dd352b6e5, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/21b4e9c1f38d4f71ad21e87c482e61f5, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/fb4a9b73dc5b422f893a64013d35206c, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/65ed1de3885a4a2f992de81ba1d63c83, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/1028e2447cfd48ada0057579afd0dedd, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/79676ba569a24d26876a783323473788, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/55727c6063034330a5fbc06cb8b2a991, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/d0aa8f6cc3e84f70b55c0c1991844808, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/57331c87d015481f9a152927af1962ef, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/9e24bab4733c4ff4a3e0cb2ae0c959d4, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/249a0a66cde5448aa5d9cd286462955e, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/84bdcacac63e4018a50c3917b91ec4a3, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/9d485cc543c94f31bde1a6a376e60578, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/4f9139655c4743a2a58317a443240a46, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/810f9f0a939d455a844a664980f64156, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/f42ea8ab90334ea485f0f550f0f07c1d, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/2cdf4312a5ef415dab0b3ac0f561388e, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/defeaf1adc4c4f7b9325b1c040e34587] to archive 2024-11-25T03:38:48,771 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T03:38:48,772 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/b474b691a1cb4e8883d8511a7ecdd6ca to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/b474b691a1cb4e8883d8511a7ecdd6ca 2024-11-25T03:38:48,772 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/5bddac04ab7f448f85f14f3dd352b6e5 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/5bddac04ab7f448f85f14f3dd352b6e5 2024-11-25T03:38:48,773 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/21b4e9c1f38d4f71ad21e87c482e61f5 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/21b4e9c1f38d4f71ad21e87c482e61f5 2024-11-25T03:38:48,774 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/fb4a9b73dc5b422f893a64013d35206c to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/fb4a9b73dc5b422f893a64013d35206c 2024-11-25T03:38:48,774 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/65ed1de3885a4a2f992de81ba1d63c83 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/65ed1de3885a4a2f992de81ba1d63c83 2024-11-25T03:38:48,775 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/1028e2447cfd48ada0057579afd0dedd to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/1028e2447cfd48ada0057579afd0dedd 2024-11-25T03:38:48,776 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/79676ba569a24d26876a783323473788 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/79676ba569a24d26876a783323473788 2024-11-25T03:38:48,777 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/55727c6063034330a5fbc06cb8b2a991 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/55727c6063034330a5fbc06cb8b2a991 2024-11-25T03:38:48,778 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/d0aa8f6cc3e84f70b55c0c1991844808 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/d0aa8f6cc3e84f70b55c0c1991844808 2024-11-25T03:38:48,779 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/57331c87d015481f9a152927af1962ef to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/57331c87d015481f9a152927af1962ef 2024-11-25T03:38:48,780 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/9e24bab4733c4ff4a3e0cb2ae0c959d4 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/9e24bab4733c4ff4a3e0cb2ae0c959d4 2024-11-25T03:38:48,780 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/249a0a66cde5448aa5d9cd286462955e to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/249a0a66cde5448aa5d9cd286462955e 2024-11-25T03:38:48,781 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/84bdcacac63e4018a50c3917b91ec4a3 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/84bdcacac63e4018a50c3917b91ec4a3 2024-11-25T03:38:48,782 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/9d485cc543c94f31bde1a6a376e60578 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/9d485cc543c94f31bde1a6a376e60578 2024-11-25T03:38:48,783 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/4f9139655c4743a2a58317a443240a46 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/4f9139655c4743a2a58317a443240a46 2024-11-25T03:38:48,784 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/810f9f0a939d455a844a664980f64156 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/810f9f0a939d455a844a664980f64156 2024-11-25T03:38:48,784 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/f42ea8ab90334ea485f0f550f0f07c1d to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/f42ea8ab90334ea485f0f550f0f07c1d 2024-11-25T03:38:48,785 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/2cdf4312a5ef415dab0b3ac0f561388e to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/2cdf4312a5ef415dab0b3ac0f561388e 2024-11-25T03:38:48,786 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/defeaf1adc4c4f7b9325b1c040e34587 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/defeaf1adc4c4f7b9325b1c040e34587 2024-11-25T03:38:48,788 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/c3e4165eb99643639fdb5496dceee90b, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/9ff74abb6f5d4f67ad7f793b0a27e727, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/fc7ce2ea177647a1bb9eb734f907e63e, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/2256708ac72f4d189ab6dff8bd6d1c89, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/e325b1051af64b1bb0056307ab3b46a7, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/682720e13a0f4f03acfd406a85cde4c8, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/2b65c52e9fe14a4db1735e8c68b74c25, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/56e2e7ab20ee463a9d21e5242871ad44, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/ebdfcf1e175f4987832e16dc4fea20ff, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/d62cf6bad0f346298cafe99f8fead3b6, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/f6ab1f10e69e4b1e9f9d4e7e208c3973, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/d1ce8015cee242f1b84f2757c258d85b, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/9146c05379d24409950cb5a3a800ca6e, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/720dade7194d439c9b76d19585997714, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/591123d5e02a4e25acd417cce7f6aeeb, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/9fe8b6c9f04744a980106beda0b09ac3, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/eca909f133f84d229964b5b7fe70fc55, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/c3b80897663e472da4f1cb08dee19664, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/60a1e35c3f724a39baf26de9a5046342] to archive 2024-11-25T03:38:48,789 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T03:38:48,790 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/c3e4165eb99643639fdb5496dceee90b to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/c3e4165eb99643639fdb5496dceee90b 2024-11-25T03:38:48,791 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/9ff74abb6f5d4f67ad7f793b0a27e727 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/9ff74abb6f5d4f67ad7f793b0a27e727 2024-11-25T03:38:48,791 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/fc7ce2ea177647a1bb9eb734f907e63e to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/fc7ce2ea177647a1bb9eb734f907e63e 2024-11-25T03:38:48,792 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/2256708ac72f4d189ab6dff8bd6d1c89 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/2256708ac72f4d189ab6dff8bd6d1c89 2024-11-25T03:38:48,793 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/e325b1051af64b1bb0056307ab3b46a7 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/e325b1051af64b1bb0056307ab3b46a7 2024-11-25T03:38:48,794 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/682720e13a0f4f03acfd406a85cde4c8 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/682720e13a0f4f03acfd406a85cde4c8 2024-11-25T03:38:48,795 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/2b65c52e9fe14a4db1735e8c68b74c25 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/2b65c52e9fe14a4db1735e8c68b74c25 2024-11-25T03:38:48,795 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/56e2e7ab20ee463a9d21e5242871ad44 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/56e2e7ab20ee463a9d21e5242871ad44 2024-11-25T03:38:48,796 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/ebdfcf1e175f4987832e16dc4fea20ff to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/ebdfcf1e175f4987832e16dc4fea20ff 2024-11-25T03:38:48,797 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/d62cf6bad0f346298cafe99f8fead3b6 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/d62cf6bad0f346298cafe99f8fead3b6 2024-11-25T03:38:48,798 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/f6ab1f10e69e4b1e9f9d4e7e208c3973 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/f6ab1f10e69e4b1e9f9d4e7e208c3973 2024-11-25T03:38:48,798 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/d1ce8015cee242f1b84f2757c258d85b to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/d1ce8015cee242f1b84f2757c258d85b 2024-11-25T03:38:48,799 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/9146c05379d24409950cb5a3a800ca6e to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/9146c05379d24409950cb5a3a800ca6e 2024-11-25T03:38:48,800 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/720dade7194d439c9b76d19585997714 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/720dade7194d439c9b76d19585997714 2024-11-25T03:38:48,800 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/591123d5e02a4e25acd417cce7f6aeeb to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/591123d5e02a4e25acd417cce7f6aeeb 2024-11-25T03:38:48,801 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/9fe8b6c9f04744a980106beda0b09ac3 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/9fe8b6c9f04744a980106beda0b09ac3 2024-11-25T03:38:48,802 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/eca909f133f84d229964b5b7fe70fc55 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/eca909f133f84d229964b5b7fe70fc55 2024-11-25T03:38:48,803 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/c3b80897663e472da4f1cb08dee19664 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/c3b80897663e472da4f1cb08dee19664 2024-11-25T03:38:48,803 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/60a1e35c3f724a39baf26de9a5046342 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/60a1e35c3f724a39baf26de9a5046342 2024-11-25T03:38:48,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-25T03:38:48,878 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:38:48,878 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-25T03:38:48,878 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-11-25T03:38:48,878 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-25T03:38:48,878 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 31 2024-11-25T03:38:48,878 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-11-25T03:38:48,878 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-25T03:38:48,878 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-25T03:38:48,878 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-25T03:38:48,878 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1171 2024-11-25T03:38:48,878 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3507 rows 2024-11-25T03:38:48,878 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1166 2024-11-25T03:38:48,878 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3497 rows 2024-11-25T03:38:48,878 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1157 2024-11-25T03:38:48,878 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3470 rows 2024-11-25T03:38:48,878 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1169 2024-11-25T03:38:48,878 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3500 rows 2024-11-25T03:38:48,878 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1166 2024-11-25T03:38:48,878 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3491 rows 2024-11-25T03:38:48,878 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T03:38:48,878 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T03:38:48,878 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:48,878 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:48,878 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:38:48,879 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-25T03:38:48,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-25T03:38:48,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-25T03:38:48,882 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732505928882"}]},"ts":"1732505928882"} 2024-11-25T03:38:48,883 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-25T03:38:48,883 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-25T03:38:48,884 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T03:38:48,885 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=629be9b3b266d56b676cd5a9595f19cd, UNASSIGN}] 2024-11-25T03:38:48,885 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=629be9b3b266d56b676cd5a9595f19cd, UNASSIGN 2024-11-25T03:38:48,886 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=629be9b3b266d56b676cd5a9595f19cd, regionState=CLOSING, regionLocation=38a5d2077268,34881,1732505806588 2024-11-25T03:38:48,887 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=629be9b3b266d56b676cd5a9595f19cd, UNASSIGN because future has completed 2024-11-25T03:38:48,887 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T03:38:48,887 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; CloseRegionProcedure 629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588}] 2024-11-25T03:38:48,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-25T03:38:49,039 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(122): Close 629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:49,039 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T03:38:49,039 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1722): Closing 629be9b3b266d56b676cd5a9595f19cd, disabling compactions & flushes 2024-11-25T03:38:49,039 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:49,039 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:49,039 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. after waiting 0 ms 2024-11-25T03:38:49,039 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:49,039 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(2902): Flushing 629be9b3b266d56b676cd5a9595f19cd 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-25T03:38:49,045 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411250bab4ab88019483686d78ad84d19b05a_629be9b3b266d56b676cd5a9595f19cd is 50, key is test_row_0/A:col10/1732505926198/Put/seqid=0 2024-11-25T03:38:49,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742228_1404 (size=12454) 2024-11-25T03:38:49,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-25T03:38:49,449 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:49,452 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411250bab4ab88019483686d78ad84d19b05a_629be9b3b266d56b676cd5a9595f19cd to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411250bab4ab88019483686d78ad84d19b05a_629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:49,453 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/b494be97c3a04b8988410d465698df56, store: [table=TestAcidGuarantees family=A region=629be9b3b266d56b676cd5a9595f19cd] 2024-11-25T03:38:49,453 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/b494be97c3a04b8988410d465698df56 is 175, key is test_row_0/A:col10/1732505926198/Put/seqid=0 2024-11-25T03:38:49,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742229_1405 (size=31255) 2024-11-25T03:38:49,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-25T03:38:49,857 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=298, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/b494be97c3a04b8988410d465698df56 2024-11-25T03:38:49,862 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/b19fff615e3347e4abf266eb801d406e is 50, key is test_row_0/B:col10/1732505926198/Put/seqid=0 2024-11-25T03:38:49,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742230_1406 (size=12301) 2024-11-25T03:38:50,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-25T03:38:50,266 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/b19fff615e3347e4abf266eb801d406e 2024-11-25T03:38:50,271 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/2f049284cc58434e8a03274acc3f54a6 is 50, key is test_row_0/C:col10/1732505926198/Put/seqid=0 2024-11-25T03:38:50,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742231_1407 (size=12301) 2024-11-25T03:38:50,675 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/2f049284cc58434e8a03274acc3f54a6 2024-11-25T03:38:50,679 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/A/b494be97c3a04b8988410d465698df56 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/b494be97c3a04b8988410d465698df56 2024-11-25T03:38:50,682 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/b494be97c3a04b8988410d465698df56, entries=150, sequenceid=298, filesize=30.5 K 2024-11-25T03:38:50,683 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/B/b19fff615e3347e4abf266eb801d406e as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/b19fff615e3347e4abf266eb801d406e 2024-11-25T03:38:50,686 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/b19fff615e3347e4abf266eb801d406e, entries=150, sequenceid=298, filesize=12.0 K 2024-11-25T03:38:50,687 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/.tmp/C/2f049284cc58434e8a03274acc3f54a6 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/2f049284cc58434e8a03274acc3f54a6 2024-11-25T03:38:50,690 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/2f049284cc58434e8a03274acc3f54a6, entries=150, sequenceid=298, filesize=12.0 K 2024-11-25T03:38:50,691 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 629be9b3b266d56b676cd5a9595f19cd in 1652ms, sequenceid=298, compaction requested=true 2024-11-25T03:38:50,695 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/recovered.edits/301.seqid, newMaxSeqId=301, maxSeqId=4 2024-11-25T03:38:50,695 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. 2024-11-25T03:38:50,695 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1676): Region close journal for 629be9b3b266d56b676cd5a9595f19cd: Waiting for close lock at 1732505929039Running coprocessor pre-close hooks at 1732505929039Disabling compacts and flushes for region at 1732505929039Disabling writes for close at 1732505929039Obtaining lock to block concurrent updates at 1732505929039Preparing flush snapshotting stores in 629be9b3b266d56b676cd5a9595f19cd at 1732505929039Finished memstore snapshotting TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732505929039Flushing stores of TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd. at 1732505929040 (+1 ms)Flushing 629be9b3b266d56b676cd5a9595f19cd/A: creating writer at 1732505929040Flushing mob file 629be9b3b266d56b676cd5a9595f19cd/A: appending metadata at 1732505929045 (+5 ms)Flushing mob file 629be9b3b266d56b676cd5a9595f19cd/A: closing flushed file at 1732505929045Flushing 629be9b3b266d56b676cd5a9595f19cd/A: appending metadata at 1732505929453 (+408 ms)Flushing 629be9b3b266d56b676cd5a9595f19cd/A: closing flushed file at 1732505929453Flushing 629be9b3b266d56b676cd5a9595f19cd/B: creating writer at 1732505929860 (+407 ms)Flushing 629be9b3b266d56b676cd5a9595f19cd/B: appending metadata at 1732505929862 (+2 ms)Flushing 629be9b3b266d56b676cd5a9595f19cd/B: closing flushed file at 1732505929862Flushing 629be9b3b266d56b676cd5a9595f19cd/C: creating writer at 1732505930269 (+407 ms)Flushing 629be9b3b266d56b676cd5a9595f19cd/C: appending metadata at 1732505930271 (+2 ms)Flushing 629be9b3b266d56b676cd5a9595f19cd/C: closing flushed file at 1732505930271Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59abe749: reopening flushed file at 1732505930679 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f0be4c5: reopening flushed file at 1732505930683 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@64738472: reopening flushed file at 1732505930686 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 629be9b3b266d56b676cd5a9595f19cd in 1652ms, sequenceid=298, compaction requested=true at 1732505930691 (+5 ms)Writing region close event to WAL at 1732505930692 (+1 ms)Running coprocessor post-close hooks at 1732505930695 (+3 ms)Closed at 1732505930695 2024-11-25T03:38:50,697 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(157): Closed 629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:50,697 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=629be9b3b266d56b676cd5a9595f19cd, regionState=CLOSED 2024-11-25T03:38:50,698 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=112, ppid=111, state=RUNNABLE, hasLock=false; CloseRegionProcedure 629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 because future has completed 2024-11-25T03:38:50,700 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-25T03:38:50,700 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseRegionProcedure 629be9b3b266d56b676cd5a9595f19cd, server=38a5d2077268,34881,1732505806588 in 1.8120 sec 2024-11-25T03:38:50,702 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=111, resume processing ppid=110 2024-11-25T03:38:50,702 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, ppid=110, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=629be9b3b266d56b676cd5a9595f19cd, UNASSIGN in 1.8150 sec 2024-11-25T03:38:50,703 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-25T03:38:50,704 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8180 sec 2024-11-25T03:38:50,705 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732505930704"}]},"ts":"1732505930704"} 2024-11-25T03:38:50,706 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-25T03:38:50,706 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-25T03:38:50,707 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8270 sec 2024-11-25T03:38:51,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-25T03:38:51,028 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:38:51,028 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-25T03:38:51,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:38:51,030 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=113, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:38:51,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-25T03:38:51,030 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=113, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:38:51,032 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:51,034 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A, FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B, FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C, FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/recovered.edits] 2024-11-25T03:38:51,036 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/884fdc82575941a08467f052d4a6ca2c to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/884fdc82575941a08467f052d4a6ca2c 2024-11-25T03:38:51,036 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/b494be97c3a04b8988410d465698df56 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/b494be97c3a04b8988410d465698df56 2024-11-25T03:38:51,037 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/f3c34f75bb9a4fec9f79c7305a8d82b3 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/A/f3c34f75bb9a4fec9f79c7305a8d82b3 2024-11-25T03:38:51,040 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/3759326503e24290b13c7dae39bb67e5 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/3759326503e24290b13c7dae39bb67e5 2024-11-25T03:38:51,040 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/557813bc507a47d3952e7a617a6309b8 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/557813bc507a47d3952e7a617a6309b8 2024-11-25T03:38:51,041 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/b19fff615e3347e4abf266eb801d406e to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/B/b19fff615e3347e4abf266eb801d406e 2024-11-25T03:38:51,043 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/2f049284cc58434e8a03274acc3f54a6 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/2f049284cc58434e8a03274acc3f54a6 2024-11-25T03:38:51,044 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/e94e5d943cde4a9a9b9ef3bf4cbcc828 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/e94e5d943cde4a9a9b9ef3bf4cbcc828 2024-11-25T03:38:51,045 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/f8a8ee40bf1e4e1ab83f416d53afcce8 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/C/f8a8ee40bf1e4e1ab83f416d53afcce8 2024-11-25T03:38:51,047 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/recovered.edits/301.seqid to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd/recovered.edits/301.seqid 2024-11-25T03:38:51,047 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:51,047 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-25T03:38:51,047 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-25T03:38:51,048 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-25T03:38:51,050 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125025b4c5a0851480da92cb2aa33567dae_629be9b3b266d56b676cd5a9595f19cd to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125025b4c5a0851480da92cb2aa33567dae_629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:51,051 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112504d5f24fc0064a968546939b5c1fd145_629be9b3b266d56b676cd5a9595f19cd to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112504d5f24fc0064a968546939b5c1fd145_629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:51,052 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112505ac05cd01f2461e96531b929639ee38_629be9b3b266d56b676cd5a9595f19cd to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112505ac05cd01f2461e96531b929639ee38_629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:51,053 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411250bab4ab88019483686d78ad84d19b05a_629be9b3b266d56b676cd5a9595f19cd to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411250bab4ab88019483686d78ad84d19b05a_629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:51,054 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411252a5930f528cf4312ba96718ab7abcc87_629be9b3b266d56b676cd5a9595f19cd to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411252a5930f528cf4312ba96718ab7abcc87_629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:51,055 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125421b312bb1524015b01a4334a433d3e1_629be9b3b266d56b676cd5a9595f19cd to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125421b312bb1524015b01a4334a433d3e1_629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:51,055 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112550455c003b724638bee16168fba916e3_629be9b3b266d56b676cd5a9595f19cd to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112550455c003b724638bee16168fba916e3_629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:51,056 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112552cdfd0c4ca743e999dbe58c67288bda_629be9b3b266d56b676cd5a9595f19cd to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112552cdfd0c4ca743e999dbe58c67288bda_629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:51,057 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411256aa8592636364b8a96a58910f4341129_629be9b3b266d56b676cd5a9595f19cd to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411256aa8592636364b8a96a58910f4341129_629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:51,058 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411257a1fd224af0248f59a35c795dd2a42fd_629be9b3b266d56b676cd5a9595f19cd to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411257a1fd224af0248f59a35c795dd2a42fd_629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:51,059 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411257edc0726dd3546e8873ecf5d7cf0ea72_629be9b3b266d56b676cd5a9595f19cd to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411257edc0726dd3546e8873ecf5d7cf0ea72_629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:51,060 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125901635e9ad7e4705bc14c23c120ae216_629be9b3b266d56b676cd5a9595f19cd to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125901635e9ad7e4705bc14c23c120ae216_629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:51,060 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125b37cfde2adcc428dbabd8dad0b85ef73_629be9b3b266d56b676cd5a9595f19cd to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125b37cfde2adcc428dbabd8dad0b85ef73_629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:51,061 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125be9031e7a9f94ec6bf0297878b8fbfe1_629be9b3b266d56b676cd5a9595f19cd to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125be9031e7a9f94ec6bf0297878b8fbfe1_629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:51,062 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125cf405a6833484c6c87af07054d7225f9_629be9b3b266d56b676cd5a9595f19cd to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125cf405a6833484c6c87af07054d7225f9_629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:51,063 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125d12ff2930cf34884bcbe8d53aa4cb6cb_629be9b3b266d56b676cd5a9595f19cd to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125d12ff2930cf34884bcbe8d53aa4cb6cb_629be9b3b266d56b676cd5a9595f19cd 2024-11-25T03:38:51,063 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-25T03:38:51,065 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=113, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:38:51,067 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-25T03:38:51,068 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-25T03:38:51,069 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=113, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:38:51,069 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-25T03:38:51,069 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732505931069"}]},"ts":"9223372036854775807"} 2024-11-25T03:38:51,071 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-25T03:38:51,071 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 629be9b3b266d56b676cd5a9595f19cd, NAME => 'TestAcidGuarantees,,1732505902481.629be9b3b266d56b676cd5a9595f19cd.', STARTKEY => '', ENDKEY => ''}] 2024-11-25T03:38:51,071 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-25T03:38:51,071 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732505931071"}]},"ts":"9223372036854775807"} 2024-11-25T03:38:51,072 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-25T03:38:51,073 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=113, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:38:51,073 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 45 msec 2024-11-25T03:38:51,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-25T03:38:51,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-25T03:38:51,137 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:38:51,148 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=242 (was 238) - Thread LEAK? -, OpenFileDescriptor=457 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1321 (was 1943), ProcessCount=11 (was 11), AvailableMemoryMB=6679 (was 7182) 2024-11-25T03:38:51,156 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=242, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=1321, ProcessCount=11, AvailableMemoryMB=6679 2024-11-25T03:38:51,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T03:38:51,157 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T03:38:51,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=114, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-25T03:38:51,159 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T03:38:51,159 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:51,159 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 114 2024-11-25T03:38:51,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-25T03:38:51,159 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T03:38:51,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742232_1408 (size=959) 2024-11-25T03:38:51,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-25T03:38:51,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-25T03:38:51,566 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3 2024-11-25T03:38:51,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742233_1409 (size=53) 2024-11-25T03:38:51,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-25T03:38:51,971 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:38:51,971 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 502b5a21115d6420c81225a0df454848, disabling compactions & flushes 2024-11-25T03:38:51,971 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:38:51,971 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:38:51,971 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. after waiting 0 ms 2024-11-25T03:38:51,971 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:38:51,971 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:38:51,971 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 502b5a21115d6420c81225a0df454848: Waiting for close lock at 1732505931971Disabling compacts and flushes for region at 1732505931971Disabling writes for close at 1732505931971Writing region close event to WAL at 1732505931971Closed at 1732505931971 2024-11-25T03:38:51,972 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T03:38:51,972 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732505931972"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732505931972"}]},"ts":"1732505931972"} 2024-11-25T03:38:51,974 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T03:38:51,974 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T03:38:51,975 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732505931974"}]},"ts":"1732505931974"} 2024-11-25T03:38:51,976 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-25T03:38:51,976 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=502b5a21115d6420c81225a0df454848, ASSIGN}] 2024-11-25T03:38:51,977 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=502b5a21115d6420c81225a0df454848, ASSIGN 2024-11-25T03:38:51,978 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=502b5a21115d6420c81225a0df454848, ASSIGN; state=OFFLINE, location=38a5d2077268,34881,1732505806588; forceNewPlan=false, retain=false 2024-11-25T03:38:52,128 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=502b5a21115d6420c81225a0df454848, regionState=OPENING, regionLocation=38a5d2077268,34881,1732505806588 2024-11-25T03:38:52,130 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=502b5a21115d6420c81225a0df454848, ASSIGN because future has completed 2024-11-25T03:38:52,130 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure 502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588}] 2024-11-25T03:38:52,284 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:38:52,284 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7752): Opening region: {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:38:52,285 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 502b5a21115d6420c81225a0df454848 2024-11-25T03:38:52,285 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:38:52,285 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7794): checking encryption for 502b5a21115d6420c81225a0df454848 2024-11-25T03:38:52,285 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7797): checking classloading for 502b5a21115d6420c81225a0df454848 2024-11-25T03:38:52,286 INFO [StoreOpener-502b5a21115d6420c81225a0df454848-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 502b5a21115d6420c81225a0df454848 2024-11-25T03:38:52,287 INFO [StoreOpener-502b5a21115d6420c81225a0df454848-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 502b5a21115d6420c81225a0df454848 columnFamilyName A 2024-11-25T03:38:52,287 DEBUG [StoreOpener-502b5a21115d6420c81225a0df454848-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:52,287 INFO [StoreOpener-502b5a21115d6420c81225a0df454848-1 {}] regionserver.HStore(327): Store=502b5a21115d6420c81225a0df454848/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:38:52,287 INFO [StoreOpener-502b5a21115d6420c81225a0df454848-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 502b5a21115d6420c81225a0df454848 2024-11-25T03:38:52,288 INFO [StoreOpener-502b5a21115d6420c81225a0df454848-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 502b5a21115d6420c81225a0df454848 columnFamilyName B 2024-11-25T03:38:52,288 DEBUG [StoreOpener-502b5a21115d6420c81225a0df454848-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:52,289 INFO [StoreOpener-502b5a21115d6420c81225a0df454848-1 {}] regionserver.HStore(327): Store=502b5a21115d6420c81225a0df454848/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:38:52,289 INFO [StoreOpener-502b5a21115d6420c81225a0df454848-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 502b5a21115d6420c81225a0df454848 2024-11-25T03:38:52,289 INFO [StoreOpener-502b5a21115d6420c81225a0df454848-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 502b5a21115d6420c81225a0df454848 columnFamilyName C 2024-11-25T03:38:52,289 DEBUG [StoreOpener-502b5a21115d6420c81225a0df454848-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:52,290 INFO [StoreOpener-502b5a21115d6420c81225a0df454848-1 {}] regionserver.HStore(327): Store=502b5a21115d6420c81225a0df454848/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:38:52,290 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1038): replaying wal for 502b5a21115d6420c81225a0df454848 2024-11-25T03:38:52,290 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848 2024-11-25T03:38:52,290 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848 2024-11-25T03:38:52,291 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1048): stopping wal replay for 502b5a21115d6420c81225a0df454848 2024-11-25T03:38:52,291 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1060): Cleaning up temporary data for 502b5a21115d6420c81225a0df454848 2024-11-25T03:38:52,291 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T03:38:52,292 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1093): writing seq id for 502b5a21115d6420c81225a0df454848 2024-11-25T03:38:52,293 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:38:52,294 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1114): Opened 502b5a21115d6420c81225a0df454848; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67371462, jitterRate=0.003913015127182007}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T03:38:52,294 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 502b5a21115d6420c81225a0df454848 2024-11-25T03:38:52,294 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1006): Region open journal for 502b5a21115d6420c81225a0df454848: Running coprocessor pre-open hook at 1732505932285Writing region info on filesystem at 1732505932285Initializing all the Stores at 1732505932285Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732505932285Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732505932286 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732505932286Cleaning up temporary data from old regions at 1732505932291 (+5 ms)Running coprocessor post-open hooks at 1732505932294 (+3 ms)Region opened successfully at 1732505932294 2024-11-25T03:38:52,295 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., pid=116, masterSystemTime=1732505932281 2024-11-25T03:38:52,296 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:38:52,296 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:38:52,297 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=502b5a21115d6420c81225a0df454848, regionState=OPEN, openSeqNum=2, regionLocation=38a5d2077268,34881,1732505806588 2024-11-25T03:38:52,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-25T03:38:52,298 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure 502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 because future has completed 2024-11-25T03:38:52,300 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-25T03:38:52,300 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; OpenRegionProcedure 502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 in 169 msec 2024-11-25T03:38:52,301 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-11-25T03:38:52,301 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=502b5a21115d6420c81225a0df454848, ASSIGN in 324 msec 2024-11-25T03:38:52,302 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T03:38:52,302 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732505932302"}]},"ts":"1732505932302"} 2024-11-25T03:38:52,303 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-25T03:38:52,304 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T03:38:52,305 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1470 sec 2024-11-25T03:38:53,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-25T03:38:53,308 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:38:53,309 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72417d0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:53,309 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:38:53,309 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:38:53,310 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:38:53,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:38:53,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:38:53,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32ffd93e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:53,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:38:53,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:38:53,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:53,312 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36908, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:38:53,312 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fb8e6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:53,312 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:38:53,313 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:38:53,313 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:38:53,314 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35690, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:38:53,315 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T03:38:53,315 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 38a5d2077268,42243,1732505805780 2024-11-25T03:38:53,315 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@14798539 2024-11-25T03:38:53,315 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T03:38:53,316 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36918, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T03:38:53,317 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3692791b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:53,317 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:38:53,317 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:38:53,318 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:38:53,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:38:53,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:38:53,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@339368bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:53,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:38:53,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:38:53,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:53,319 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36922, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:38:53,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bedaef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:53,320 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b791feb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:53,320 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:38:53,320 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:38:53,320 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:38:53,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:38:53,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:38:53,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ed4cf4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:53,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:38:53,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:38:53,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:53,321 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36948, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:38:53,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e43432c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:53,322 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6da2e0be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:53,322 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:38:53,322 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:38:53,323 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:38:53,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:38:53,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:38:53,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66ba4591, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:53,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:38:53,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:38:53,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:53,324 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36962, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:38:53,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75d8f90f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:53,325 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fc361f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:53,325 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:38:53,325 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:38:53,326 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:38:53,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:38:53,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:38:53,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e8a9aa8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:53,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:38:53,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:38:53,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:53,327 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36974, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:38:53,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@688589a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:53,328 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@259a56de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:53,328 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:38:53,328 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:38:53,328 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:38:53,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:38:53,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:38:53,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35878420, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:53,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:38:53,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:38:53,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:53,329 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36998, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:38:53,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3202e312, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:53,330 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ae789b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:53,330 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:38:53,330 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:38:53,331 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:38:53,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:38:53,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:38:53,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4106c21b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:53,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:38:53,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:38:53,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:53,332 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37018, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:38:53,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@423c5afa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:53,333 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b7b3b89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:53,333 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:38:53,333 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:38:53,334 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:38:53,334 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:38:53,334 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:38:53,334 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21d403d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:53,334 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:38:53,334 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:38:53,334 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:53,335 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37026, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:38:53,335 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ba849c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:53,336 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e04f49c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:53,336 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:38:53,336 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:38:53,337 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:38:53,337 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:38:53,337 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:38:53,337 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bfd8b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:53,337 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:38:53,337 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:38:53,337 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:53,338 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37042, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:38:53,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4019ab3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:53,339 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32ca199c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:53,339 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:38:53,339 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:38:53,339 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:38:53,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:38:53,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:38:53,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f8524a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:53,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:38:53,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:38:53,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:53,340 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37054, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:38:53,341 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b43d3a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:53,341 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d70caa4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:53,341 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:38:53,341 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:38:53,342 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:38:53,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:38:53,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:38:53,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52a5f24f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:53,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:38:53,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:38:53,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:38:53,343 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37076, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:38:53,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1481f3e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:38:53,344 DEBUG [Thread-1857 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:38:53,344 DEBUG [Thread-1870 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:38:53,344 DEBUG [Thread-1865 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:38:53,344 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:38:53,344 DEBUG [Thread-1874 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:38:53,345 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:38:53,345 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:38:53,345 DEBUG [Thread-1861 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:38:53,345 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:38:53,345 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:38:53,345 DEBUG [Thread-1863 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:38:53,345 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:38:53,345 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:38:53,346 DEBUG [Thread-1872 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:38:53,346 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35720, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:38:53,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:38:53,346 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35704, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:38:53,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:38:53,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:38:53,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:38:53,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-25T03:38:53,346 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35722, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:38:53,346 DEBUG [Thread-1876 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:38:53,347 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:38:53,347 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35728, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:38:53,347 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:38:53,347 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35738, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:38:53,347 DEBUG [Thread-1868 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:38:53,347 DEBUG [Thread-1859 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:38:53,347 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:38:53,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-25T03:38:53,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:38:53,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:38:53,348 DEBUG [Thread-1868 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:38:53,348 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:38:53,348 DEBUG [Thread-1868 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:38:53,349 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:38:53,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2] 2024-11-25T03:38:53,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2] 2024-11-25T03:38:53,349 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35746, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:38:53,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:38:53,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2] 2024-11-25T03:38:53,350 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:38:53,350 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35742, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:38:53,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2] 2024-11-25T03:38:53,350 DEBUG [Thread-1859 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:38:53,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2] 2024-11-25T03:38:53,350 DEBUG [Thread-1859 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:38:53,351 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35762, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:38:53,351 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35768, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:38:53,351 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35776, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:38:53,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2] 2024-11-25T03:38:53,353 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2] 2024-11-25T03:38:53,353 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2] 2024-11-25T03:38:53,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2] 2024-11-25T03:38:53,354 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2] 2024-11-25T03:38:53,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 502b5a21115d6420c81225a0df454848 2024-11-25T03:38:53,361 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:38:53,361 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 502b5a21115d6420c81225a0df454848 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-25T03:38:53,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/867146695ed545839e8e4005e179805a is 50, key is test_row_0/A:col10/1732505933359/Put/seqid=0 2024-11-25T03:38:53,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742234_1410 (size=16681) 2024-11-25T03:38:53,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:53,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35704 deadline: 1732505993408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:53,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:53,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:53,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1732505993413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:53,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:53,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:53,419 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:53,419 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:53,419 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:53,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:53,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35728 deadline: 1732505993414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:53,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:53,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:53,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35768 deadline: 1732505993415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:53,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:53,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:53,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:53,425 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:53,425 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:53,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35738 deadline: 1732505993419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:53,425 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:53,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:53,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:53,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:53,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-25T03:38:53,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T03:38:53,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:38:53,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. as already flushing 2024-11-25T03:38:53,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:38:53,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:53,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:53,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:53,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:53,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35704 deadline: 1732505993527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:53,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:53,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1732505993527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:53,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:53,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:53,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:53,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:53,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35728 deadline: 1732505993527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:53,532 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:53,532 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:53,532 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:53,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:53,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:53,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:53,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35768 deadline: 1732505993527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:53,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:53,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:53,532 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:53,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35738 deadline: 1732505993527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:53,532 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:53,532 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:53,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:53,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:53,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:53,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T03:38:53,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:38:53,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. as already flushing 2024-11-25T03:38:53,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:38:53,658 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:53,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:53,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:53,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-25T03:38:53,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:53,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35704 deadline: 1732505993737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:53,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:53,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:53,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:53,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:53,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1732505993737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:53,740 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:53,741 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:53,741 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:53,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:53,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35768 deadline: 1732505993737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:53,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:53,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35728 deadline: 1732505993737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:53,746 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:53,746 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:53,746 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:53,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:53,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35738 deadline: 1732505993737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:53,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:53,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:53,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:53,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:53,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:53,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:53,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/867146695ed545839e8e4005e179805a 2024-11-25T03:38:53,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T03:38:53,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:38:53,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. as already flushing 2024-11-25T03:38:53,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:38:53,811 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:53,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:53,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:53,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/b5e64a2f085b4165878e5ab5ab7d57b4 is 50, key is test_row_0/B:col10/1732505933359/Put/seqid=0 2024-11-25T03:38:53,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742235_1411 (size=12001) 2024-11-25T03:38:53,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T03:38:53,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:38:53,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. as already flushing 2024-11-25T03:38:53,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:38:53,965 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:53,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:53,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:53,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-25T03:38:54,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:54,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35704 deadline: 1732505994047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:54,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:54,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:54,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:54,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1732505994047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:54,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:54,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:54,054 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:54,055 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:54,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35768 deadline: 1732505994047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:54,055 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:54,055 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:54,055 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:54,055 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:54,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:54,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35728 deadline: 1732505994057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:54,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:54,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:54,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35738 deadline: 1732505994057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:54,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:54,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:54,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:54,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:54,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:54,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T03:38:54,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:38:54,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. as already flushing 2024-11-25T03:38:54,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:38:54,118 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:54,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:54,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:54,223 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/b5e64a2f085b4165878e5ab5ab7d57b4 2024-11-25T03:38:54,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/61b01d34d42041acae85a514a22605e7 is 50, key is test_row_0/C:col10/1732505933359/Put/seqid=0 2024-11-25T03:38:54,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742236_1412 (size=12001) 2024-11-25T03:38:54,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T03:38:54,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:38:54,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. as already flushing 2024-11-25T03:38:54,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:38:54,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:54,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:54,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:54,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T03:38:54,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:38:54,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. as already flushing 2024-11-25T03:38:54,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:38:54,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:54,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:54,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:54,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-25T03:38:54,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:54,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35704 deadline: 1732505994557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:54,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:54,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:54,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:54,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:54,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1732505994567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:54,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:54,569 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:54,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35768 deadline: 1732505994567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:54,569 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:54,569 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:54,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:54,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:54,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:54,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T03:38:54,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:38:54,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. as already flushing 2024-11-25T03:38:54,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:38:54,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:54,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:54,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:54,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:54,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35728 deadline: 1732505994578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:54,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:54,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:54,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:54,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35738 deadline: 1732505994578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:54,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:54,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:54,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:54,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:54,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/61b01d34d42041acae85a514a22605e7 2024-11-25T03:38:54,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/867146695ed545839e8e4005e179805a as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/867146695ed545839e8e4005e179805a 2024-11-25T03:38:54,658 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/867146695ed545839e8e4005e179805a, entries=250, sequenceid=12, filesize=16.3 K 2024-11-25T03:38:54,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/b5e64a2f085b4165878e5ab5ab7d57b4 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/b5e64a2f085b4165878e5ab5ab7d57b4 2024-11-25T03:38:54,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/b5e64a2f085b4165878e5ab5ab7d57b4, entries=150, sequenceid=12, filesize=11.7 K 2024-11-25T03:38:54,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/61b01d34d42041acae85a514a22605e7 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/61b01d34d42041acae85a514a22605e7 2024-11-25T03:38:54,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/61b01d34d42041acae85a514a22605e7, entries=150, sequenceid=12, filesize=11.7 K 2024-11-25T03:38:54,668 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 502b5a21115d6420c81225a0df454848 in 1307ms, sequenceid=12, compaction requested=false 2024-11-25T03:38:54,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 502b5a21115d6420c81225a0df454848: 2024-11-25T03:38:54,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T03:38:54,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:38:54,729 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 502b5a21115d6420c81225a0df454848 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-25T03:38:54,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/10c789d1aebb4595992fcbd7d141f22c is 50, key is test_row_0/A:col10/1732505933413/Put/seqid=0 2024-11-25T03:38:54,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742237_1413 (size=12001) 2024-11-25T03:38:55,138 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/10c789d1aebb4595992fcbd7d141f22c 2024-11-25T03:38:55,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/911a6515ba45499fbbfb65cdb3d4410b is 50, key is test_row_0/B:col10/1732505933413/Put/seqid=0 2024-11-25T03:38:55,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742238_1414 (size=12001) 2024-11-25T03:38:55,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-25T03:38:55,555 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/911a6515ba45499fbbfb65cdb3d4410b 2024-11-25T03:38:55,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/10d63eac96ec4c73b2f3b9c11f8fa152 is 50, key is test_row_0/C:col10/1732505933413/Put/seqid=0 2024-11-25T03:38:55,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742239_1415 (size=12001) 2024-11-25T03:38:55,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 502b5a21115d6420c81225a0df454848 2024-11-25T03:38:55,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. as already flushing 2024-11-25T03:38:55,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:55,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35704 deadline: 1732505995592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:55,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:55,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:55,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:55,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:55,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1732505995593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:55,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:55,601 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:55,601 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:55,601 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:55,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35768 deadline: 1732505995595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:55,602 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:55,602 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:55,602 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:55,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:55,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35728 deadline: 1732505995597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:55,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:55,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35738 deadline: 1732505995597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:55,606 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,34881,1732505806588 for 'test_row_0' in 502b5a21115d6420c81225a0df454848 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:38:55,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:55,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:55,606 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,34881,1732505806588 for 'test_row_1' in 502b5a21115d6420c81225a0df454848 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:38:55,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:55,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:55,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:55,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:55,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:55,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35704 deadline: 1732505995697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:55,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:55,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:55,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:55,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:55,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1732505995707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:55,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:55,717 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:55,717 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:55,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35768 deadline: 1732505995707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:55,717 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:55,717 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:55,717 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:55,717 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:55,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:55,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35704 deadline: 1732505995917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:55,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:55,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:55,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:55,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:55,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1732505995927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:55,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:55,935 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:55,936 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:55,936 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:55,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35768 deadline: 1732505995927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:55,936 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:55,936 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:55,936 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:55,971 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/10d63eac96ec4c73b2f3b9c11f8fa152 2024-11-25T03:38:55,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/10c789d1aebb4595992fcbd7d141f22c as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/10c789d1aebb4595992fcbd7d141f22c 2024-11-25T03:38:55,979 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/10c789d1aebb4595992fcbd7d141f22c, entries=150, sequenceid=37, filesize=11.7 K 2024-11-25T03:38:55,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/911a6515ba45499fbbfb65cdb3d4410b as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/911a6515ba45499fbbfb65cdb3d4410b 2024-11-25T03:38:55,983 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/911a6515ba45499fbbfb65cdb3d4410b, entries=150, sequenceid=37, filesize=11.7 K 2024-11-25T03:38:55,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/10d63eac96ec4c73b2f3b9c11f8fa152 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/10d63eac96ec4c73b2f3b9c11f8fa152 2024-11-25T03:38:55,987 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/10d63eac96ec4c73b2f3b9c11f8fa152, entries=150, sequenceid=37, filesize=11.7 K 2024-11-25T03:38:55,988 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 502b5a21115d6420c81225a0df454848 in 1258ms, sequenceid=37, compaction requested=false 2024-11-25T03:38:55,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 502b5a21115d6420c81225a0df454848: 2024-11-25T03:38:55,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:38:55,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-25T03:38:55,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-25T03:38:55,991 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-25T03:38:55,991 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6410 sec 2024-11-25T03:38:55,992 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 2.6470 sec 2024-11-25T03:38:56,216 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T03:38:56,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 502b5a21115d6420c81225a0df454848 2024-11-25T03:38:56,232 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:38:56,232 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 502b5a21115d6420c81225a0df454848 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-25T03:38:56,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/24f5bae460c2407a8ba9d16d45ea22c4 is 50, key is test_row_0/A:col10/1732505936231/Put/seqid=0 2024-11-25T03:38:56,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742240_1416 (size=14341) 2024-11-25T03:38:56,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:56,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35704 deadline: 1732505996312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:56,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:56,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:56,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:56,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:56,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35768 deadline: 1732505996313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:56,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:56,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1732505996313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:56,325 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:56,325 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:56,325 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:56,325 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:56,325 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:56,325 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:56,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:56,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35704 deadline: 1732505996417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:56,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:56,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:56,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:56,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:56,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35768 deadline: 1732505996428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:56,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:56,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1732505996428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:56,437 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:56,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:56,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:56,438 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:56,438 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:56,438 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:56,644 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/24f5bae460c2407a8ba9d16d45ea22c4 2024-11-25T03:38:56,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:56,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35704 deadline: 1732505996637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:56,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:56,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:56,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:56,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/6d0cd51effd84a1ea4ac8776df27d822 is 50, key is test_row_0/B:col10/1732505936231/Put/seqid=0 2024-11-25T03:38:56,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742241_1417 (size=12001) 2024-11-25T03:38:56,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:56,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35768 deadline: 1732505996647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:56,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:56,656 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:56,656 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:56,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1732505996648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:56,656 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:56,656 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:56,656 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:56,656 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:56,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:56,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35704 deadline: 1732505996957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:56,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:56,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:56,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:56,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:56,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35768 deadline: 1732505996967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:56,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:56,975 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:56,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1732505996967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:56,975 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:56,975 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:56,976 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:56,976 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:56,976 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:57,054 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/6d0cd51effd84a1ea4ac8776df27d822 2024-11-25T03:38:57,061 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/80951d4328fa42359641f637bebe2b76 is 50, key is test_row_0/C:col10/1732505936231/Put/seqid=0 2024-11-25T03:38:57,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742242_1418 (size=12001) 2024-11-25T03:38:57,465 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/80951d4328fa42359641f637bebe2b76 2024-11-25T03:38:57,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/24f5bae460c2407a8ba9d16d45ea22c4 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/24f5bae460c2407a8ba9d16d45ea22c4 2024-11-25T03:38:57,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/24f5bae460c2407a8ba9d16d45ea22c4, entries=200, sequenceid=49, filesize=14.0 K 2024-11-25T03:38:57,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/6d0cd51effd84a1ea4ac8776df27d822 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/6d0cd51effd84a1ea4ac8776df27d822 2024-11-25T03:38:57,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/6d0cd51effd84a1ea4ac8776df27d822, entries=150, sequenceid=49, filesize=11.7 K 2024-11-25T03:38:57,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/80951d4328fa42359641f637bebe2b76 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/80951d4328fa42359641f637bebe2b76 2024-11-25T03:38:57,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/80951d4328fa42359641f637bebe2b76, entries=150, sequenceid=49, filesize=11.7 K 2024-11-25T03:38:57,480 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 502b5a21115d6420c81225a0df454848 in 1248ms, sequenceid=49, compaction requested=true 2024-11-25T03:38:57,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 502b5a21115d6420c81225a0df454848: 2024-11-25T03:38:57,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 502b5a21115d6420c81225a0df454848:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:38:57,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:38:57,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 502b5a21115d6420c81225a0df454848:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:38:57,481 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:38:57,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:38:57,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 502b5a21115d6420c81225a0df454848:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:38:57,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:38:57,481 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:38:57,481 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:38:57,482 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:38:57,482 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): 502b5a21115d6420c81225a0df454848/A is initiating minor compaction (all files) 2024-11-25T03:38:57,482 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 502b5a21115d6420c81225a0df454848/B is initiating minor compaction (all files) 2024-11-25T03:38:57,482 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 502b5a21115d6420c81225a0df454848/A in TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:38:57,482 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 502b5a21115d6420c81225a0df454848/B in TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:38:57,482 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/867146695ed545839e8e4005e179805a, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/10c789d1aebb4595992fcbd7d141f22c, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/24f5bae460c2407a8ba9d16d45ea22c4] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp, totalSize=42.0 K 2024-11-25T03:38:57,482 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/b5e64a2f085b4165878e5ab5ab7d57b4, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/911a6515ba45499fbbfb65cdb3d4410b, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/6d0cd51effd84a1ea4ac8776df27d822] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp, totalSize=35.2 K 2024-11-25T03:38:57,482 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting b5e64a2f085b4165878e5ab5ab7d57b4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732505933350 2024-11-25T03:38:57,482 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 867146695ed545839e8e4005e179805a, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732505933350 2024-11-25T03:38:57,482 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 911a6515ba45499fbbfb65cdb3d4410b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732505933411 2024-11-25T03:38:57,482 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10c789d1aebb4595992fcbd7d141f22c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732505933411 2024-11-25T03:38:57,483 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d0cd51effd84a1ea4ac8776df27d822, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732505935590 2024-11-25T03:38:57,483 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24f5bae460c2407a8ba9d16d45ea22c4, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732505935581 2024-11-25T03:38:57,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 502b5a21115d6420c81225a0df454848 2024-11-25T03:38:57,486 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:38:57,486 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 502b5a21115d6420c81225a0df454848 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-25T03:38:57,497 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/2b66e9b80eb643cc9809dfca1de47920 is 50, key is test_row_0/A:col10/1732505937484/Put/seqid=0 2024-11-25T03:38:57,499 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 502b5a21115d6420c81225a0df454848#B#compaction#348 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:38:57,499 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/4bcbad7586df42b0acd295b972c2a872 is 50, key is test_row_0/B:col10/1732505936231/Put/seqid=0 2024-11-25T03:38:57,501 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 502b5a21115d6420c81225a0df454848#A#compaction#349 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:38:57,502 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/c6b6f01ded5f4c60aa7d4093e19c7d51 is 50, key is test_row_0/A:col10/1732505936231/Put/seqid=0 2024-11-25T03:38:57,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742243_1419 (size=12001) 2024-11-25T03:38:57,505 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/2b66e9b80eb643cc9809dfca1de47920 2024-11-25T03:38:57,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742244_1420 (size=12104) 2024-11-25T03:38:57,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:57,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35704 deadline: 1732505997509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:57,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:57,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:57,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:57,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-25T03:38:57,518 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:38:57,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:57,518 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:38:57,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1732505997515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:57,519 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:57,519 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:57,519 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:57,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-25T03:38:57,520 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:38:57,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-25T03:38:57,521 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:38:57,521 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:38:57,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:57,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35768 deadline: 1732505997517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:57,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742245_1421 (size=12104) 2024-11-25T03:38:57,530 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:57,531 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:57,531 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:57,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/58d598f8a233470685fb1d3298123dd3 is 50, key is test_row_0/B:col10/1732505937484/Put/seqid=0 2024-11-25T03:38:57,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742246_1422 (size=12001) 2024-11-25T03:38:57,536 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/58d598f8a233470685fb1d3298123dd3 2024-11-25T03:38:57,537 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/4bcbad7586df42b0acd295b972c2a872 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/4bcbad7586df42b0acd295b972c2a872 2024-11-25T03:38:57,542 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 502b5a21115d6420c81225a0df454848/B of 502b5a21115d6420c81225a0df454848 into 4bcbad7586df42b0acd295b972c2a872(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:38:57,542 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 502b5a21115d6420c81225a0df454848: 2024-11-25T03:38:57,542 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., storeName=502b5a21115d6420c81225a0df454848/B, priority=13, startTime=1732505937481; duration=0sec 2024-11-25T03:38:57,542 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:38:57,542 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 502b5a21115d6420c81225a0df454848:B 2024-11-25T03:38:57,542 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:38:57,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/b7611f4544e74ed995edfda4943c4f36 is 50, key is test_row_0/C:col10/1732505937484/Put/seqid=0 2024-11-25T03:38:57,543 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:38:57,544 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 502b5a21115d6420c81225a0df454848/C is initiating minor compaction (all files) 2024-11-25T03:38:57,544 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 502b5a21115d6420c81225a0df454848/C in TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:38:57,544 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/61b01d34d42041acae85a514a22605e7, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/10d63eac96ec4c73b2f3b9c11f8fa152, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/80951d4328fa42359641f637bebe2b76] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp, totalSize=35.2 K 2024-11-25T03:38:57,544 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 61b01d34d42041acae85a514a22605e7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732505933350 2024-11-25T03:38:57,545 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 10d63eac96ec4c73b2f3b9c11f8fa152, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732505933411 2024-11-25T03:38:57,545 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 80951d4328fa42359641f637bebe2b76, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732505935590 2024-11-25T03:38:57,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742247_1423 (size=12001) 2024-11-25T03:38:57,551 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 502b5a21115d6420c81225a0df454848#C#compaction#352 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:38:57,552 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/87965423dafa40b498cb23d0b2fa084c is 50, key is test_row_0/C:col10/1732505936231/Put/seqid=0 2024-11-25T03:38:57,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742248_1424 (size=12104) 2024-11-25T03:38:57,562 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/87965423dafa40b498cb23d0b2fa084c as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/87965423dafa40b498cb23d0b2fa084c 2024-11-25T03:38:57,566 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 502b5a21115d6420c81225a0df454848/C of 502b5a21115d6420c81225a0df454848 into 87965423dafa40b498cb23d0b2fa084c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:38:57,566 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 502b5a21115d6420c81225a0df454848: 2024-11-25T03:38:57,566 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., storeName=502b5a21115d6420c81225a0df454848/C, priority=13, startTime=1732505937481; duration=0sec 2024-11-25T03:38:57,566 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:38:57,566 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 502b5a21115d6420c81225a0df454848:C 2024-11-25T03:38:57,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:57,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35728 deadline: 1732505997617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:57,627 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,34881,1732505806588 for 'test_row_0' in 502b5a21115d6420c81225a0df454848 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:38:57,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:57,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:57,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:57,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-25T03:38:57,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:57,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35738 deadline: 1732505997628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:57,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:57,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35704 deadline: 1732505997628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:57,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:57,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:57,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1732505997628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:57,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:57,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:57,632 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,34881,1732505806588 for 'test_row_1' in 502b5a21115d6420c81225a0df454848 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:38:57,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:57,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:57,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:57,632 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:57,632 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:57,632 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:57,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:57,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35768 deadline: 1732505997637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:57,643 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:57,643 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:57,643 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:57,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-25T03:38:57,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:38:57,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. as already flushing 2024-11-25T03:38:57,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:38:57,673 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:57,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:57,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:57,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-25T03:38:57,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:38:57,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. as already flushing 2024-11-25T03:38:57,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:38:57,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:57,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:57,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:57,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-25T03:38:57,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:57,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35704 deadline: 1732505997838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:57,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:57,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:57,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1732505997838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:57,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:57,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:57,843 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:57,843 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:57,843 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:57,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:57,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35768 deadline: 1732505997847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:57,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:57,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:57,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:57,918 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/c6b6f01ded5f4c60aa7d4093e19c7d51 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/c6b6f01ded5f4c60aa7d4093e19c7d51 2024-11-25T03:38:57,922 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 502b5a21115d6420c81225a0df454848/A of 502b5a21115d6420c81225a0df454848 into c6b6f01ded5f4c60aa7d4093e19c7d51(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:38:57,922 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 502b5a21115d6420c81225a0df454848: 2024-11-25T03:38:57,922 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., storeName=502b5a21115d6420c81225a0df454848/A, priority=13, startTime=1732505937481; duration=0sec 2024-11-25T03:38:57,922 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:38:57,922 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 502b5a21115d6420c81225a0df454848:A 2024-11-25T03:38:57,948 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/b7611f4544e74ed995edfda4943c4f36 2024-11-25T03:38:57,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/2b66e9b80eb643cc9809dfca1de47920 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/2b66e9b80eb643cc9809dfca1de47920 2024-11-25T03:38:57,956 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/2b66e9b80eb643cc9809dfca1de47920, entries=150, sequenceid=75, filesize=11.7 K 2024-11-25T03:38:57,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/58d598f8a233470685fb1d3298123dd3 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/58d598f8a233470685fb1d3298123dd3 2024-11-25T03:38:57,959 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/58d598f8a233470685fb1d3298123dd3, entries=150, sequenceid=75, filesize=11.7 K 2024-11-25T03:38:57,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/b7611f4544e74ed995edfda4943c4f36 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/b7611f4544e74ed995edfda4943c4f36 2024-11-25T03:38:57,963 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/b7611f4544e74ed995edfda4943c4f36, entries=150, sequenceid=75, filesize=11.7 K 2024-11-25T03:38:57,964 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 502b5a21115d6420c81225a0df454848 in 478ms, sequenceid=75, compaction requested=false 2024-11-25T03:38:57,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 502b5a21115d6420c81225a0df454848: 2024-11-25T03:38:57,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-25T03:38:57,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:38:57,979 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 502b5a21115d6420c81225a0df454848 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-25T03:38:57,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/d20c677351d84b00888e0da1555e7a40 is 50, key is test_row_0/A:col10/1732505937515/Put/seqid=0 2024-11-25T03:38:57,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742249_1425 (size=9657) 2024-11-25T03:38:58,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-25T03:38:58,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 502b5a21115d6420c81225a0df454848 2024-11-25T03:38:58,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. as already flushing 2024-11-25T03:38:58,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:58,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35768 deadline: 1732505998213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:58,223 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:58,223 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:58,223 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:58,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:58,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35704 deadline: 1732505998215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:58,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:58,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:58,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:58,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:58,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1732505998223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:58,229 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:58,229 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:58,229 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:58,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:58,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35768 deadline: 1732505998327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:58,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:58,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:58,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:58,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:58,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35704 deadline: 1732505998338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:58,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:58,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:58,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1732505998338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:58,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:58,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:58,345 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:58,345 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:58,345 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:58,388 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/d20c677351d84b00888e0da1555e7a40 2024-11-25T03:38:58,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/53e9ea64c32f444fa2dd5b6391badaf6 is 50, key is test_row_0/B:col10/1732505937515/Put/seqid=0 2024-11-25T03:38:58,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742250_1426 (size=9657) 2024-11-25T03:38:58,398 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/53e9ea64c32f444fa2dd5b6391badaf6 2024-11-25T03:38:58,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/0b92f8b53b39470dab67621c7bb0868e is 50, key is test_row_0/C:col10/1732505937515/Put/seqid=0 2024-11-25T03:38:58,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742251_1427 (size=9657) 2024-11-25T03:38:58,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:58,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35768 deadline: 1732505998547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:58,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:58,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35704 deadline: 1732505998547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:58,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:58,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:58,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:58,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:58,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:58,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:58,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:58,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1732505998547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:58,555 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:58,555 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:58,555 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:58,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-25T03:38:58,807 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/0b92f8b53b39470dab67621c7bb0868e 2024-11-25T03:38:58,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/d20c677351d84b00888e0da1555e7a40 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/d20c677351d84b00888e0da1555e7a40 2024-11-25T03:38:58,815 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/d20c677351d84b00888e0da1555e7a40, entries=100, sequenceid=88, filesize=9.4 K 2024-11-25T03:38:58,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/53e9ea64c32f444fa2dd5b6391badaf6 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/53e9ea64c32f444fa2dd5b6391badaf6 2024-11-25T03:38:58,819 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/53e9ea64c32f444fa2dd5b6391badaf6, entries=100, sequenceid=88, filesize=9.4 K 2024-11-25T03:38:58,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/0b92f8b53b39470dab67621c7bb0868e as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/0b92f8b53b39470dab67621c7bb0868e 2024-11-25T03:38:58,824 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/0b92f8b53b39470dab67621c7bb0868e, entries=100, sequenceid=88, filesize=9.4 K 2024-11-25T03:38:58,825 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 502b5a21115d6420c81225a0df454848 in 846ms, sequenceid=88, compaction requested=true 2024-11-25T03:38:58,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 502b5a21115d6420c81225a0df454848: 2024-11-25T03:38:58,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:38:58,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-25T03:38:58,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-25T03:38:58,828 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-25T03:38:58,828 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3050 sec 2024-11-25T03:38:58,829 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.3100 sec 2024-11-25T03:38:58,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 502b5a21115d6420c81225a0df454848 2024-11-25T03:38:58,863 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:38:58,864 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 502b5a21115d6420c81225a0df454848 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-25T03:38:58,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/7d7933614a8c47449bb3a2fe135c3738 is 50, key is test_row_0/A:col10/1732505938862/Put/seqid=0 2024-11-25T03:38:58,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742252_1428 (size=14341) 2024-11-25T03:38:58,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:58,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35768 deadline: 1732505998878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:58,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:58,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35704 deadline: 1732505998887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:58,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:58,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:58,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:58,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:58,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1732505998889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:58,899 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:58,899 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:58,899 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:59,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:59,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35768 deadline: 1732505998997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:59,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:59,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35704 deadline: 1732505998997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:59,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:59,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:59,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:59,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:59,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:59,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:59,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:59,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1732505999007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:59,014 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:59,014 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:59,014 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:59,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:59,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35768 deadline: 1732505999207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:59,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:59,211 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:59,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35704 deadline: 1732505999207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:59,211 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:59,211 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:59,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:59,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:59,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:59,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:59,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1732505999218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:59,226 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:59,226 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:59,226 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:59,274 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/7d7933614a8c47449bb3a2fe135c3738 2024-11-25T03:38:59,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/76ee702f29b24a08933d75bc990d85c3 is 50, key is test_row_0/B:col10/1732505938862/Put/seqid=0 2024-11-25T03:38:59,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742253_1429 (size=12001) 2024-11-25T03:38:59,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:59,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35768 deadline: 1732505999518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:59,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:59,522 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:59,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35704 deadline: 1732505999518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:59,522 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:59,522 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:59,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:59,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:59,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:59,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:38:59,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1732505999537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:38:59,543 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:59,543 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:38:59,543 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:38:59,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-25T03:38:59,668 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:38:59,668 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:38:59,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-25T03:38:59,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-25T03:38:59,670 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:38:59,670 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:38:59,671 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:38:59,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/76ee702f29b24a08933d75bc990d85c3 2024-11-25T03:38:59,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/9e776cbca2fb4120b18b2fd46ea0caba is 50, key is test_row_0/C:col10/1732505938862/Put/seqid=0 2024-11-25T03:38:59,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742254_1430 (size=12001) 2024-11-25T03:38:59,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-25T03:38:59,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-25T03:38:59,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:38:59,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. as already flushing 2024-11-25T03:38:59,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:38:59,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:59,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:59,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:59,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-25T03:38:59,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:38:59,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. as already flushing 2024-11-25T03:38:59,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:38:59,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:59,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:59,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:59,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-25T03:39:00,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:00,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35768 deadline: 1732506000027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:00,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:00,032 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:00,032 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:00,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35704 deadline: 1732506000027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:00,032 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:00,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:00,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:00,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:00,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:00,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1732506000047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:00,053 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:00,053 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:00,053 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:00,109 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/9e776cbca2fb4120b18b2fd46ea0caba 2024-11-25T03:39:00,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/7d7933614a8c47449bb3a2fe135c3738 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/7d7933614a8c47449bb3a2fe135c3738 2024-11-25T03:39:00,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/7d7933614a8c47449bb3a2fe135c3738, entries=200, sequenceid=115, filesize=14.0 K 2024-11-25T03:39:00,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/76ee702f29b24a08933d75bc990d85c3 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/76ee702f29b24a08933d75bc990d85c3 2024-11-25T03:39:00,121 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/76ee702f29b24a08933d75bc990d85c3, entries=150, sequenceid=115, filesize=11.7 K 2024-11-25T03:39:00,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/9e776cbca2fb4120b18b2fd46ea0caba as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/9e776cbca2fb4120b18b2fd46ea0caba 2024-11-25T03:39:00,129 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/9e776cbca2fb4120b18b2fd46ea0caba, entries=150, sequenceid=115, filesize=11.7 K 2024-11-25T03:39:00,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-25T03:39:00,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:00,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. as already flushing 2024-11-25T03:39:00,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:00,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:00,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:00,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:00,135 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 502b5a21115d6420c81225a0df454848 in 1272ms, sequenceid=115, compaction requested=true 2024-11-25T03:39:00,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 502b5a21115d6420c81225a0df454848: 2024-11-25T03:39:00,135 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:39:00,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 502b5a21115d6420c81225a0df454848:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:39:00,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:39:00,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 502b5a21115d6420c81225a0df454848:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:39:00,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:39:00,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 502b5a21115d6420c81225a0df454848:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:39:00,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:39:00,135 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:39:00,137 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45763 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:39:00,138 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 502b5a21115d6420c81225a0df454848/B is initiating minor compaction (all files) 2024-11-25T03:39:00,138 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 502b5a21115d6420c81225a0df454848/B in TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:00,138 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/4bcbad7586df42b0acd295b972c2a872, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/58d598f8a233470685fb1d3298123dd3, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/53e9ea64c32f444fa2dd5b6391badaf6, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/76ee702f29b24a08933d75bc990d85c3] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp, totalSize=44.7 K 2024-11-25T03:39:00,138 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48103 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:39:00,138 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): 502b5a21115d6420c81225a0df454848/A is initiating minor compaction (all files) 2024-11-25T03:39:00,138 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 502b5a21115d6420c81225a0df454848/A in TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:00,138 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/c6b6f01ded5f4c60aa7d4093e19c7d51, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/2b66e9b80eb643cc9809dfca1de47920, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/d20c677351d84b00888e0da1555e7a40, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/7d7933614a8c47449bb3a2fe135c3738] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp, totalSize=47.0 K 2024-11-25T03:39:00,139 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bcbad7586df42b0acd295b972c2a872, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732505935590 2024-11-25T03:39:00,139 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6b6f01ded5f4c60aa7d4093e19c7d51, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732505935590 2024-11-25T03:39:00,139 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 58d598f8a233470685fb1d3298123dd3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732505936296 2024-11-25T03:39:00,139 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b66e9b80eb643cc9809dfca1de47920, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732505936296 2024-11-25T03:39:00,140 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 53e9ea64c32f444fa2dd5b6391badaf6, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732505937514 2024-11-25T03:39:00,140 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting d20c677351d84b00888e0da1555e7a40, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732505937514 2024-11-25T03:39:00,140 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 76ee702f29b24a08933d75bc990d85c3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732505938210 2024-11-25T03:39:00,140 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d7933614a8c47449bb3a2fe135c3738, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732505938210 2024-11-25T03:39:00,154 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 502b5a21115d6420c81225a0df454848#B#compaction#360 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:39:00,155 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/27e0f8fc0036462ebe4e3abd72b4be56 is 50, key is test_row_0/B:col10/1732505938862/Put/seqid=0 2024-11-25T03:39:00,157 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 502b5a21115d6420c81225a0df454848#A#compaction#359 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:39:00,158 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/84624cbac1ed4ebfa508bf04846a1454 is 50, key is test_row_0/A:col10/1732505938862/Put/seqid=0 2024-11-25T03:39:00,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742255_1431 (size=12241) 2024-11-25T03:39:00,190 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/27e0f8fc0036462ebe4e3abd72b4be56 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/27e0f8fc0036462ebe4e3abd72b4be56 2024-11-25T03:39:00,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742256_1432 (size=12241) 2024-11-25T03:39:00,197 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 502b5a21115d6420c81225a0df454848/B of 502b5a21115d6420c81225a0df454848 into 27e0f8fc0036462ebe4e3abd72b4be56(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:39:00,197 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 502b5a21115d6420c81225a0df454848: 2024-11-25T03:39:00,197 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., storeName=502b5a21115d6420c81225a0df454848/B, priority=12, startTime=1732505940135; duration=0sec 2024-11-25T03:39:00,197 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:39:00,197 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 502b5a21115d6420c81225a0df454848:B 2024-11-25T03:39:00,197 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:39:00,198 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45763 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:39:00,198 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 502b5a21115d6420c81225a0df454848/C is initiating minor compaction (all files) 2024-11-25T03:39:00,198 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 502b5a21115d6420c81225a0df454848/C in TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:00,199 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/87965423dafa40b498cb23d0b2fa084c, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/b7611f4544e74ed995edfda4943c4f36, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/0b92f8b53b39470dab67621c7bb0868e, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/9e776cbca2fb4120b18b2fd46ea0caba] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp, totalSize=44.7 K 2024-11-25T03:39:00,199 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 87965423dafa40b498cb23d0b2fa084c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732505935590 2024-11-25T03:39:00,199 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting b7611f4544e74ed995edfda4943c4f36, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732505936296 2024-11-25T03:39:00,200 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b92f8b53b39470dab67621c7bb0868e, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732505937514 2024-11-25T03:39:00,200 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e776cbca2fb4120b18b2fd46ea0caba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732505938210 2024-11-25T03:39:00,200 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/84624cbac1ed4ebfa508bf04846a1454 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/84624cbac1ed4ebfa508bf04846a1454 2024-11-25T03:39:00,206 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 502b5a21115d6420c81225a0df454848/A of 502b5a21115d6420c81225a0df454848 into 84624cbac1ed4ebfa508bf04846a1454(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:39:00,206 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 502b5a21115d6420c81225a0df454848: 2024-11-25T03:39:00,206 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., storeName=502b5a21115d6420c81225a0df454848/A, priority=12, startTime=1732505940135; duration=0sec 2024-11-25T03:39:00,206 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:39:00,206 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 502b5a21115d6420c81225a0df454848:A 2024-11-25T03:39:00,210 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 502b5a21115d6420c81225a0df454848#C#compaction#361 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:39:00,211 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/011296115c204a66a8c575136e984999 is 50, key is test_row_0/C:col10/1732505938862/Put/seqid=0 2024-11-25T03:39:00,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742257_1433 (size=12241) 2024-11-25T03:39:00,241 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/011296115c204a66a8c575136e984999 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/011296115c204a66a8c575136e984999 2024-11-25T03:39:00,250 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 502b5a21115d6420c81225a0df454848/C of 502b5a21115d6420c81225a0df454848 into 011296115c204a66a8c575136e984999(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:39:00,250 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 502b5a21115d6420c81225a0df454848: 2024-11-25T03:39:00,250 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., storeName=502b5a21115d6420c81225a0df454848/C, priority=12, startTime=1732505940135; duration=0sec 2024-11-25T03:39:00,250 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:39:00,250 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 502b5a21115d6420c81225a0df454848:C 2024-11-25T03:39:00,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-25T03:39:00,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:00,288 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 502b5a21115d6420c81225a0df454848 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-25T03:39:00,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/9e6dfc9223da4882989e4786eb5ae2ee is 50, key is test_row_0/A:col10/1732505938864/Put/seqid=0 2024-11-25T03:39:00,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-25T03:39:00,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742258_1434 (size=12001) 2024-11-25T03:39:00,302 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/9e6dfc9223da4882989e4786eb5ae2ee 2024-11-25T03:39:00,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/462b6f87ba7446488db30b7d8f13a38d is 50, key is test_row_0/B:col10/1732505938864/Put/seqid=0 2024-11-25T03:39:00,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742259_1435 (size=12001) 2024-11-25T03:39:00,349 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/462b6f87ba7446488db30b7d8f13a38d 2024-11-25T03:39:00,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/120f35ef20bf44c5a7d614d4f87353ca is 50, key is test_row_0/C:col10/1732505938864/Put/seqid=0 2024-11-25T03:39:00,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742260_1436 (size=12001) 2024-11-25T03:39:00,375 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/120f35ef20bf44c5a7d614d4f87353ca 2024-11-25T03:39:00,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/9e6dfc9223da4882989e4786eb5ae2ee as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/9e6dfc9223da4882989e4786eb5ae2ee 2024-11-25T03:39:00,386 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/9e6dfc9223da4882989e4786eb5ae2ee, entries=150, sequenceid=128, filesize=11.7 K 2024-11-25T03:39:00,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/462b6f87ba7446488db30b7d8f13a38d as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/462b6f87ba7446488db30b7d8f13a38d 2024-11-25T03:39:00,393 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/462b6f87ba7446488db30b7d8f13a38d, entries=150, sequenceid=128, filesize=11.7 K 2024-11-25T03:39:00,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/120f35ef20bf44c5a7d614d4f87353ca as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/120f35ef20bf44c5a7d614d4f87353ca 2024-11-25T03:39:00,402 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/120f35ef20bf44c5a7d614d4f87353ca, entries=150, sequenceid=128, filesize=11.7 K 2024-11-25T03:39:00,403 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=0 B/0 for 502b5a21115d6420c81225a0df454848 in 115ms, sequenceid=128, compaction requested=false 2024-11-25T03:39:00,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 502b5a21115d6420c81225a0df454848: 2024-11-25T03:39:00,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:00,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-25T03:39:00,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-25T03:39:00,408 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-25T03:39:00,408 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 734 msec 2024-11-25T03:39:00,409 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 740 msec 2024-11-25T03:39:00,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-25T03:39:00,807 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:39:00,808 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:39:00,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-25T03:39:00,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-25T03:39:00,810 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:39:00,811 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:39:00,811 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:39:00,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-25T03:39:00,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-25T03:39:00,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:00,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 502b5a21115d6420c81225a0df454848: 2024-11-25T03:39:00,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:00,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-25T03:39:00,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-25T03:39:00,966 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-25T03:39:00,966 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 154 msec 2024-11-25T03:39:00,968 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 159 msec 2024-11-25T03:39:01,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 502b5a21115d6420c81225a0df454848 2024-11-25T03:39:01,057 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:39:01,058 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 502b5a21115d6420c81225a0df454848 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-25T03:39:01,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/fdcb53913c32436d9c3c2bfbc407bb7d is 50, key is test_row_0/A:col10/1732505941049/Put/seqid=0 2024-11-25T03:39:01,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742261_1437 (size=12151) 2024-11-25T03:39:01,069 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/fdcb53913c32436d9c3c2bfbc407bb7d 2024-11-25T03:39:01,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/c28555b852e84629adaee83238c0ecc9 is 50, key is test_row_0/B:col10/1732505941049/Put/seqid=0 2024-11-25T03:39:01,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742262_1438 (size=12151) 2024-11-25T03:39:01,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-25T03:39:01,127 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:39:01,128 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:39:01,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-25T03:39:01,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-25T03:39:01,129 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:39:01,130 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:39:01,130 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:39:01,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:01,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1732506001136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:01,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:01,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35768 deadline: 1732506001137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:01,145 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:01,145 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:01,145 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:01,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:01,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:01,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:01,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:01,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35704 deadline: 1732506001141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:01,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:01,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:01,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:01,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-25T03:39:01,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:01,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1732506001247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:01,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:01,253 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:01,253 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:01,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35768 deadline: 1732506001247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:01,253 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:01,254 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:01,254 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:01,254 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:01,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:01,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35704 deadline: 1732506001257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:01,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:01,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:01,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:01,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-25T03:39:01,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:01,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. as already flushing 2024-11-25T03:39:01,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:01,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:01,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:01,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:01,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-25T03:39:01,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:01,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. as already flushing 2024-11-25T03:39:01,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:01,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:01,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:01,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:01,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-25T03:39:01,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:01,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1732506001457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:01,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:01,466 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:01,467 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:01,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35768 deadline: 1732506001457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:01,467 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:01,467 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:01,467 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:01,467 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:01,479 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/c28555b852e84629adaee83238c0ecc9 2024-11-25T03:39:01,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:01,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35704 deadline: 1732506001478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:01,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:01,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:01,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:01,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/2c2fa26a81fe4a28b96846948dc3a96a is 50, key is test_row_0/C:col10/1732505941049/Put/seqid=0 2024-11-25T03:39:01,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742263_1439 (size=12151) 2024-11-25T03:39:01,492 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/2c2fa26a81fe4a28b96846948dc3a96a 2024-11-25T03:39:01,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/fdcb53913c32436d9c3c2bfbc407bb7d as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/fdcb53913c32436d9c3c2bfbc407bb7d 2024-11-25T03:39:01,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/fdcb53913c32436d9c3c2bfbc407bb7d, entries=150, sequenceid=140, filesize=11.9 K 2024-11-25T03:39:01,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/c28555b852e84629adaee83238c0ecc9 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/c28555b852e84629adaee83238c0ecc9 2024-11-25T03:39:01,506 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/c28555b852e84629adaee83238c0ecc9, entries=150, sequenceid=140, filesize=11.9 K 2024-11-25T03:39:01,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/2c2fa26a81fe4a28b96846948dc3a96a as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/2c2fa26a81fe4a28b96846948dc3a96a 2024-11-25T03:39:01,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/2c2fa26a81fe4a28b96846948dc3a96a, entries=150, sequenceid=140, filesize=11.9 K 2024-11-25T03:39:01,512 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 502b5a21115d6420c81225a0df454848 in 455ms, sequenceid=140, compaction requested=true 2024-11-25T03:39:01,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 502b5a21115d6420c81225a0df454848: 2024-11-25T03:39:01,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 502b5a21115d6420c81225a0df454848:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:39:01,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:39:01,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 502b5a21115d6420c81225a0df454848:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:39:01,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:39:01,512 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:39:01,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 502b5a21115d6420c81225a0df454848:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:39:01,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:39:01,512 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:39:01,513 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:39:01,513 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:39:01,513 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): 502b5a21115d6420c81225a0df454848/A is initiating minor compaction (all files) 2024-11-25T03:39:01,513 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 502b5a21115d6420c81225a0df454848/B is initiating minor compaction (all files) 2024-11-25T03:39:01,513 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 502b5a21115d6420c81225a0df454848/B in TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:01,513 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 502b5a21115d6420c81225a0df454848/A in TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:01,513 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/27e0f8fc0036462ebe4e3abd72b4be56, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/462b6f87ba7446488db30b7d8f13a38d, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/c28555b852e84629adaee83238c0ecc9] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp, totalSize=35.5 K 2024-11-25T03:39:01,513 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/84624cbac1ed4ebfa508bf04846a1454, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/9e6dfc9223da4882989e4786eb5ae2ee, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/fdcb53913c32436d9c3c2bfbc407bb7d] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp, totalSize=35.5 K 2024-11-25T03:39:01,516 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 27e0f8fc0036462ebe4e3abd72b4be56, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732505938210 2024-11-25T03:39:01,516 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84624cbac1ed4ebfa508bf04846a1454, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732505938210 2024-11-25T03:39:01,516 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e6dfc9223da4882989e4786eb5ae2ee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732505938864 2024-11-25T03:39:01,516 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 462b6f87ba7446488db30b7d8f13a38d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732505938864 2024-11-25T03:39:01,517 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting fdcb53913c32436d9c3c2bfbc407bb7d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732505941048 2024-11-25T03:39:01,517 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting c28555b852e84629adaee83238c0ecc9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732505941048 2024-11-25T03:39:01,528 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 502b5a21115d6420c81225a0df454848#B#compaction#368 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:39:01,528 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 502b5a21115d6420c81225a0df454848#A#compaction#369 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:39:01,528 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/466dded260544d99a29d6d6bca3a53f8 is 50, key is test_row_0/B:col10/1732505941049/Put/seqid=0 2024-11-25T03:39:01,528 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/9ffe5326d6ce4c4eb668787eccd572a4 is 50, key is test_row_0/A:col10/1732505941049/Put/seqid=0 2024-11-25T03:39:01,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742265_1441 (size=12493) 2024-11-25T03:39:01,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742264_1440 (size=12493) 2024-11-25T03:39:01,540 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/466dded260544d99a29d6d6bca3a53f8 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/466dded260544d99a29d6d6bca3a53f8 2024-11-25T03:39:01,545 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 502b5a21115d6420c81225a0df454848/B of 502b5a21115d6420c81225a0df454848 into 466dded260544d99a29d6d6bca3a53f8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:39:01,545 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 502b5a21115d6420c81225a0df454848: 2024-11-25T03:39:01,545 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., storeName=502b5a21115d6420c81225a0df454848/B, priority=13, startTime=1732505941512; duration=0sec 2024-11-25T03:39:01,545 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:39:01,545 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 502b5a21115d6420c81225a0df454848:B 2024-11-25T03:39:01,545 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:39:01,546 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:39:01,546 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 502b5a21115d6420c81225a0df454848/C is initiating minor compaction (all files) 2024-11-25T03:39:01,547 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 502b5a21115d6420c81225a0df454848/C in TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:01,547 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/011296115c204a66a8c575136e984999, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/120f35ef20bf44c5a7d614d4f87353ca, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/2c2fa26a81fe4a28b96846948dc3a96a] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp, totalSize=35.5 K 2024-11-25T03:39:01,547 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 011296115c204a66a8c575136e984999, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732505938210 2024-11-25T03:39:01,548 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 120f35ef20bf44c5a7d614d4f87353ca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732505938864 2024-11-25T03:39:01,548 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c2fa26a81fe4a28b96846948dc3a96a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732505941048 2024-11-25T03:39:01,554 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 502b5a21115d6420c81225a0df454848#C#compaction#370 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:39:01,555 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/26958da9aa144e3eb7b722dd0d7847d3 is 50, key is test_row_0/C:col10/1732505941049/Put/seqid=0 2024-11-25T03:39:01,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742266_1442 (size=12493) 2024-11-25T03:39:01,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-25T03:39:01,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:01,589 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 502b5a21115d6420c81225a0df454848 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-25T03:39:01,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/674a6f804c9f4f2a8051bb20e920376f is 50, key is test_row_0/A:col10/1732505941134/Put/seqid=0 2024-11-25T03:39:01,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742267_1443 (size=12151) 2024-11-25T03:39:01,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 502b5a21115d6420c81225a0df454848 2024-11-25T03:39:01,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. as already flushing 2024-11-25T03:39:01,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:01,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35728 deadline: 1732506001679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:01,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:01,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:01,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:01,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:01,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35738 deadline: 1732506001680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:01,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:01,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:01,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:01,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-25T03:39:01,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:01,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1732506001777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:01,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:01,786 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:01,787 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:01,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35768 deadline: 1732506001777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:01,787 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:01,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:01,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:01,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:01,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:01,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35704 deadline: 1732506001788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:01,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:01,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:01,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:01,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35728 deadline: 1732506001788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:01,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:01,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:01,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:01,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:01,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:01,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35738 deadline: 1732506001797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:01,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:01,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:01,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:01,938 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/9ffe5326d6ce4c4eb668787eccd572a4 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/9ffe5326d6ce4c4eb668787eccd572a4 2024-11-25T03:39:01,941 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 502b5a21115d6420c81225a0df454848/A of 502b5a21115d6420c81225a0df454848 into 9ffe5326d6ce4c4eb668787eccd572a4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:39:01,941 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 502b5a21115d6420c81225a0df454848: 2024-11-25T03:39:01,942 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., storeName=502b5a21115d6420c81225a0df454848/A, priority=13, startTime=1732505941512; duration=0sec 2024-11-25T03:39:01,942 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:39:01,942 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 502b5a21115d6420c81225a0df454848:A 2024-11-25T03:39:01,965 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/26958da9aa144e3eb7b722dd0d7847d3 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/26958da9aa144e3eb7b722dd0d7847d3 2024-11-25T03:39:01,968 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 502b5a21115d6420c81225a0df454848/C of 502b5a21115d6420c81225a0df454848 into 26958da9aa144e3eb7b722dd0d7847d3(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:39:01,968 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 502b5a21115d6420c81225a0df454848: 2024-11-25T03:39:01,969 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., storeName=502b5a21115d6420c81225a0df454848/C, priority=13, startTime=1732505941512; duration=0sec 2024-11-25T03:39:01,969 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:39:01,969 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 502b5a21115d6420c81225a0df454848:C 2024-11-25T03:39:02,000 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/674a6f804c9f4f2a8051bb20e920376f 2024-11-25T03:39:02,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:02,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35728 deadline: 1732506001997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:02,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:02,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:02,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:02,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/2ef3f68fc2f64c908f62c43a8ec038d9 is 50, key is test_row_0/B:col10/1732505941134/Put/seqid=0 2024-11-25T03:39:02,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:02,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35738 deadline: 1732506002008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:02,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:02,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:02,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:02,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742268_1444 (size=12151) 2024-11-25T03:39:02,013 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/2ef3f68fc2f64c908f62c43a8ec038d9 2024-11-25T03:39:02,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/a00c8cf665a547c8a8a1bcf067988582 is 50, key is test_row_0/C:col10/1732505941134/Put/seqid=0 2024-11-25T03:39:02,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742269_1445 (size=12151) 2024-11-25T03:39:02,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-25T03:39:02,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:02,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1732506002297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:02,305 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:02,305 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:02,305 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:02,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:02,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35768 deadline: 1732506002297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:02,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:02,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:02,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:02,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:02,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35704 deadline: 1732506002308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:02,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:02,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:02,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:02,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:02,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35728 deadline: 1732506002317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:02,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:02,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:02,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:02,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35738 deadline: 1732506002317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:02,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:02,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:02,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:02,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:02,424 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/a00c8cf665a547c8a8a1bcf067988582 2024-11-25T03:39:02,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/674a6f804c9f4f2a8051bb20e920376f as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/674a6f804c9f4f2a8051bb20e920376f 2024-11-25T03:39:02,431 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/674a6f804c9f4f2a8051bb20e920376f, entries=150, sequenceid=166, filesize=11.9 K 2024-11-25T03:39:02,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/2ef3f68fc2f64c908f62c43a8ec038d9 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/2ef3f68fc2f64c908f62c43a8ec038d9 2024-11-25T03:39:02,435 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/2ef3f68fc2f64c908f62c43a8ec038d9, entries=150, sequenceid=166, filesize=11.9 K 2024-11-25T03:39:02,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/a00c8cf665a547c8a8a1bcf067988582 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/a00c8cf665a547c8a8a1bcf067988582 2024-11-25T03:39:02,439 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/a00c8cf665a547c8a8a1bcf067988582, entries=150, sequenceid=166, filesize=11.9 K 2024-11-25T03:39:02,440 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 502b5a21115d6420c81225a0df454848 in 851ms, sequenceid=166, compaction requested=false 2024-11-25T03:39:02,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 502b5a21115d6420c81225a0df454848: 2024-11-25T03:39:02,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:02,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-25T03:39:02,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-25T03:39:02,443 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-25T03:39:02,443 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3110 sec 2024-11-25T03:39:02,445 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.3160 sec 2024-11-25T03:39:02,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 502b5a21115d6420c81225a0df454848 2024-11-25T03:39:02,842 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:39:02,842 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 502b5a21115d6420c81225a0df454848 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-25T03:39:02,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/74d25f312513457a9080b395614dffe7 is 50, key is test_row_0/A:col10/1732505941668/Put/seqid=0 2024-11-25T03:39:02,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742270_1446 (size=14541) 2024-11-25T03:39:02,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:02,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35728 deadline: 1732506002930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:02,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:02,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35738 deadline: 1732506002934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:02,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:02,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:02,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:02,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:02,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:02,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:03,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:03,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35728 deadline: 1732506003047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:03,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:03,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35738 deadline: 1732506003048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:03,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:03,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:03,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:03,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:03,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:03,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:03,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/74d25f312513457a9080b395614dffe7 2024-11-25T03:39:03,261 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/52bf1c0783b44f3794ed8294166b9c17 is 50, key is test_row_0/B:col10/1732505941668/Put/seqid=0 2024-11-25T03:39:03,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742271_1447 (size=12151) 2024-11-25T03:39:03,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:03,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35728 deadline: 1732506003257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:03,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:03,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:03,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:03,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35738 deadline: 1732506003257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:03,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:03,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:03,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:03,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:03,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-25T03:39:03,277 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:39:03,277 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:39:03,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-25T03:39:03,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-25T03:39:03,279 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:39:03,280 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:39:03,280 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:39:03,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:03,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1732506003317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:03,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:03,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35768 deadline: 1732506003317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:03,327 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,34881,1732505806588 for 'test_row_2' in 502b5a21115d6420c81225a0df454848 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:39:03,327 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,34881,1732505806588 for 'test_row_1' in 502b5a21115d6420c81225a0df454848 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:39:03,327 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:03,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:03,328 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:03,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:03,328 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:03,328 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:03,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:03,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35704 deadline: 1732506003328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:03,337 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,34881,1732505806588 for 'test_row_0' in 502b5a21115d6420c81225a0df454848 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:39:03,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:03,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:03,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:03,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-25T03:39:03,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-25T03:39:03,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:03,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. as already flushing 2024-11-25T03:39:03,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:03,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:03,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:03,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:03,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:03,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35728 deadline: 1732506003578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:03,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:03,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35738 deadline: 1732506003578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:03,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:03,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:03,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:03,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:03,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:03,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:03,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-25T03:39:03,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:03,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. as already flushing 2024-11-25T03:39:03,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:03,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:03,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:03,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:03,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-25T03:39:03,665 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/52bf1c0783b44f3794ed8294166b9c17 2024-11-25T03:39:03,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/86250147737249259873a6303ab01547 is 50, key is test_row_0/C:col10/1732505941668/Put/seqid=0 2024-11-25T03:39:03,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742272_1448 (size=12151) 2024-11-25T03:39:03,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-25T03:39:03,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:03,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. as already flushing 2024-11-25T03:39:03,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:03,738 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:03,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:03,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:03,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-25T03:39:03,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:03,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. as already flushing 2024-11-25T03:39:03,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:03,891 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:03,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:03,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:03,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-25T03:39:04,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-25T03:39:04,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:04,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. as already flushing 2024-11-25T03:39:04,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:04,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:04,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:04,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:04,077 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/86250147737249259873a6303ab01547 2024-11-25T03:39:04,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/74d25f312513457a9080b395614dffe7 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/74d25f312513457a9080b395614dffe7 2024-11-25T03:39:04,084 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/74d25f312513457a9080b395614dffe7, entries=200, sequenceid=181, filesize=14.2 K 2024-11-25T03:39:04,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/52bf1c0783b44f3794ed8294166b9c17 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/52bf1c0783b44f3794ed8294166b9c17 2024-11-25T03:39:04,087 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/52bf1c0783b44f3794ed8294166b9c17, entries=150, sequenceid=181, filesize=11.9 K 2024-11-25T03:39:04,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/86250147737249259873a6303ab01547 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/86250147737249259873a6303ab01547 2024-11-25T03:39:04,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/86250147737249259873a6303ab01547, entries=150, sequenceid=181, filesize=11.9 K 2024-11-25T03:39:04,092 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 502b5a21115d6420c81225a0df454848 in 1249ms, sequenceid=181, compaction requested=true 2024-11-25T03:39:04,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 502b5a21115d6420c81225a0df454848: 2024-11-25T03:39:04,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 502b5a21115d6420c81225a0df454848:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:39:04,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:39:04,092 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:39:04,092 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:39:04,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 502b5a21115d6420c81225a0df454848:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:39:04,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:39:04,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 502b5a21115d6420c81225a0df454848:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:39:04,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:39:04,093 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:39:04,093 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): 502b5a21115d6420c81225a0df454848/A is initiating minor compaction (all files) 2024-11-25T03:39:04,093 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:39:04,093 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 502b5a21115d6420c81225a0df454848/A in TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:04,093 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 502b5a21115d6420c81225a0df454848/B is initiating minor compaction (all files) 2024-11-25T03:39:04,093 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/9ffe5326d6ce4c4eb668787eccd572a4, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/674a6f804c9f4f2a8051bb20e920376f, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/74d25f312513457a9080b395614dffe7] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp, totalSize=38.3 K 2024-11-25T03:39:04,093 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 502b5a21115d6420c81225a0df454848/B in TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:04,093 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/466dded260544d99a29d6d6bca3a53f8, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/2ef3f68fc2f64c908f62c43a8ec038d9, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/52bf1c0783b44f3794ed8294166b9c17] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp, totalSize=35.9 K 2024-11-25T03:39:04,093 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ffe5326d6ce4c4eb668787eccd572a4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732505941048 2024-11-25T03:39:04,094 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 466dded260544d99a29d6d6bca3a53f8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732505941048 2024-11-25T03:39:04,094 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 674a6f804c9f4f2a8051bb20e920376f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732505941125 2024-11-25T03:39:04,094 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74d25f312513457a9080b395614dffe7, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732505941668 2024-11-25T03:39:04,094 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ef3f68fc2f64c908f62c43a8ec038d9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732505941125 2024-11-25T03:39:04,094 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 52bf1c0783b44f3794ed8294166b9c17, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732505941668 2024-11-25T03:39:04,101 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 502b5a21115d6420c81225a0df454848#A#compaction#377 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:39:04,102 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/8a5a815fbd5a4213a674e7667e51053e is 50, key is test_row_0/A:col10/1732505941668/Put/seqid=0 2024-11-25T03:39:04,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 502b5a21115d6420c81225a0df454848 2024-11-25T03:39:04,103 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:39:04,103 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 502b5a21115d6420c81225a0df454848 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-25T03:39:04,104 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 502b5a21115d6420c81225a0df454848#B#compaction#378 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:39:04,105 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/a733e9b77e4c43a7a3aa97050c6a60c4 is 50, key is test_row_0/B:col10/1732505941668/Put/seqid=0 2024-11-25T03:39:04,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742273_1449 (size=12595) 2024-11-25T03:39:04,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/18837405895c492d867015148033df07 is 50, key is test_row_0/A:col10/1732505944102/Put/seqid=0 2024-11-25T03:39:04,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742274_1450 (size=12595) 2024-11-25T03:39:04,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742275_1451 (size=14541) 2024-11-25T03:39:04,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:04,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:04,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35728 deadline: 1732506004138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:04,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35738 deadline: 1732506004139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:04,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:04,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:04,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:04,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:04,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:04,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:04,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-25T03:39:04,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:04,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. as already flushing 2024-11-25T03:39:04,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:04,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:04,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:04,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:04,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:04,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35728 deadline: 1732506004257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:04,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:04,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35738 deadline: 1732506004257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:04,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:04,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:04,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:04,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:04,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:04,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:04,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-25T03:39:04,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:04,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. as already flushing 2024-11-25T03:39:04,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:04,350 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:04,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:04,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:04,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-25T03:39:04,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:04,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35728 deadline: 1732506004477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:04,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:04,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35738 deadline: 1732506004477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:04,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:04,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:04,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:04,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:04,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:04,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:04,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-25T03:39:04,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:04,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. as already flushing 2024-11-25T03:39:04,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:04,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:04,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:04,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:04,515 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/8a5a815fbd5a4213a674e7667e51053e as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/8a5a815fbd5a4213a674e7667e51053e 2024-11-25T03:39:04,519 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 502b5a21115d6420c81225a0df454848/A of 502b5a21115d6420c81225a0df454848 into 8a5a815fbd5a4213a674e7667e51053e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:39:04,519 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 502b5a21115d6420c81225a0df454848: 2024-11-25T03:39:04,519 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., storeName=502b5a21115d6420c81225a0df454848/A, priority=13, startTime=1732505944092; duration=0sec 2024-11-25T03:39:04,519 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:39:04,519 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 502b5a21115d6420c81225a0df454848:A 2024-11-25T03:39:04,519 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:39:04,520 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:39:04,520 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): 502b5a21115d6420c81225a0df454848/C is initiating minor compaction (all files) 2024-11-25T03:39:04,520 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 502b5a21115d6420c81225a0df454848/C in TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:04,520 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/26958da9aa144e3eb7b722dd0d7847d3, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/a00c8cf665a547c8a8a1bcf067988582, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/86250147737249259873a6303ab01547] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp, totalSize=35.9 K 2024-11-25T03:39:04,520 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26958da9aa144e3eb7b722dd0d7847d3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732505941048 2024-11-25T03:39:04,520 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting a00c8cf665a547c8a8a1bcf067988582, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732505941125 2024-11-25T03:39:04,521 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86250147737249259873a6303ab01547, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732505941668 2024-11-25T03:39:04,530 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/a733e9b77e4c43a7a3aa97050c6a60c4 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/a733e9b77e4c43a7a3aa97050c6a60c4 2024-11-25T03:39:04,530 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/18837405895c492d867015148033df07 2024-11-25T03:39:04,535 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 502b5a21115d6420c81225a0df454848/B of 502b5a21115d6420c81225a0df454848 into a733e9b77e4c43a7a3aa97050c6a60c4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:39:04,535 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 502b5a21115d6420c81225a0df454848: 2024-11-25T03:39:04,535 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., storeName=502b5a21115d6420c81225a0df454848/B, priority=13, startTime=1732505944092; duration=0sec 2024-11-25T03:39:04,535 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:39:04,535 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 502b5a21115d6420c81225a0df454848:B 2024-11-25T03:39:04,545 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 502b5a21115d6420c81225a0df454848#C#compaction#380 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:39:04,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/2f1dd915ee8948218fa833f21f299ee5 is 50, key is test_row_0/B:col10/1732505944102/Put/seqid=0 2024-11-25T03:39:04,546 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/d14f1d4953264608a69487c574258884 is 50, key is test_row_0/C:col10/1732505941668/Put/seqid=0 2024-11-25T03:39:04,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742276_1452 (size=12151) 2024-11-25T03:39:04,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742277_1453 (size=12595) 2024-11-25T03:39:04,556 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/d14f1d4953264608a69487c574258884 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/d14f1d4953264608a69487c574258884 2024-11-25T03:39:04,560 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 502b5a21115d6420c81225a0df454848/C of 502b5a21115d6420c81225a0df454848 into d14f1d4953264608a69487c574258884(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:39:04,560 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 502b5a21115d6420c81225a0df454848: 2024-11-25T03:39:04,560 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., storeName=502b5a21115d6420c81225a0df454848/C, priority=13, startTime=1732505944092; duration=0sec 2024-11-25T03:39:04,561 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:39:04,561 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 502b5a21115d6420c81225a0df454848:C 2024-11-25T03:39:04,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-25T03:39:04,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:04,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. as already flushing 2024-11-25T03:39:04,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:04,655 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:04,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:04,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:04,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:04,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35728 deadline: 1732506004797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:04,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:04,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:04,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:04,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35738 deadline: 1732506004797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:04,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:04,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:04,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:04,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:04,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-25T03:39:04,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:04,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. as already flushing 2024-11-25T03:39:04,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:04,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:04,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:04,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:04,952 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/2f1dd915ee8948218fa833f21f299ee5 2024-11-25T03:39:04,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/19ee9d62a219490ea319e6d843600cdd is 50, key is test_row_0/C:col10/1732505944102/Put/seqid=0 2024-11-25T03:39:04,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-25T03:39:04,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:04,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. as already flushing 2024-11-25T03:39:04,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:04,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:04,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:04,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:04,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742278_1454 (size=12151) 2024-11-25T03:39:05,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-25T03:39:05,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:05,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. as already flushing 2024-11-25T03:39:05,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:05,114 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:05,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:05,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:05,266 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-25T03:39:05,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:05,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. as already flushing 2024-11-25T03:39:05,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:05,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:05,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:05,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:05,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:05,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35728 deadline: 1732506005307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:05,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:05,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:05,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:05,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:05,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35738 deadline: 1732506005317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:05,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:05,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:05,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:05,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:05,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1732506005347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:05,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:05,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35704 deadline: 1732506005347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:05,352 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,34881,1732505806588 for 'test_row_2' in 502b5a21115d6420c81225a0df454848 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:39:05,352 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:05,352 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,34881,1732505806588 for 'test_row_0' in 502b5a21115d6420c81225a0df454848 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:39:05,352 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:05,353 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:05,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:05,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:05,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:05,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:05,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35768 deadline: 1732506005357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:05,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/19ee9d62a219490ea319e6d843600cdd 2024-11-25T03:39:05,363 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,34881,1732505806588 for 'test_row_1' in 502b5a21115d6420c81225a0df454848 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:39:05,364 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:05,364 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:05,364 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:05,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/18837405895c492d867015148033df07 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/18837405895c492d867015148033df07 2024-11-25T03:39:05,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/18837405895c492d867015148033df07, entries=200, sequenceid=205, filesize=14.2 K 2024-11-25T03:39:05,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/2f1dd915ee8948218fa833f21f299ee5 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/2f1dd915ee8948218fa833f21f299ee5 2024-11-25T03:39:05,375 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/2f1dd915ee8948218fa833f21f299ee5, entries=150, sequenceid=205, filesize=11.9 K 2024-11-25T03:39:05,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/19ee9d62a219490ea319e6d843600cdd as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/19ee9d62a219490ea319e6d843600cdd 2024-11-25T03:39:05,379 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/19ee9d62a219490ea319e6d843600cdd, entries=150, sequenceid=205, filesize=11.9 K 2024-11-25T03:39:05,380 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 502b5a21115d6420c81225a0df454848 in 1277ms, sequenceid=205, compaction requested=false 2024-11-25T03:39:05,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 502b5a21115d6420c81225a0df454848: 2024-11-25T03:39:05,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-25T03:39:05,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:05,420 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 502b5a21115d6420c81225a0df454848 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-25T03:39:05,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/308033b80bb54bfabcf73933613953db is 50, key is test_row_0/A:col10/1732505944133/Put/seqid=0 2024-11-25T03:39:05,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-25T03:39:05,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742279_1455 (size=12151) 2024-11-25T03:39:05,830 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/308033b80bb54bfabcf73933613953db 2024-11-25T03:39:05,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/3f91a930ed884c919c81095ab3fa749c is 50, key is test_row_0/B:col10/1732505944133/Put/seqid=0 2024-11-25T03:39:05,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742280_1456 (size=12151) 2024-11-25T03:39:06,249 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/3f91a930ed884c919c81095ab3fa749c 2024-11-25T03:39:06,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/be7b105c673a491ebdf838f86d655d5b is 50, key is test_row_0/C:col10/1732505944133/Put/seqid=0 2024-11-25T03:39:06,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742281_1457 (size=12151) 2024-11-25T03:39:06,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 502b5a21115d6420c81225a0df454848 2024-11-25T03:39:06,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. as already flushing 2024-11-25T03:39:06,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:06,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35728 deadline: 1732506006416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:06,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:06,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:06,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:06,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:06,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35738 deadline: 1732506006423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:06,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:06,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:06,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:06,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:06,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35728 deadline: 1732506006527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:06,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:06,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:06,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:06,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:06,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35738 deadline: 1732506006537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:06,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:06,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:06,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:06,660 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/be7b105c673a491ebdf838f86d655d5b 2024-11-25T03:39:06,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/308033b80bb54bfabcf73933613953db as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/308033b80bb54bfabcf73933613953db 2024-11-25T03:39:06,667 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/308033b80bb54bfabcf73933613953db, entries=150, sequenceid=220, filesize=11.9 K 2024-11-25T03:39:06,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/3f91a930ed884c919c81095ab3fa749c as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/3f91a930ed884c919c81095ab3fa749c 2024-11-25T03:39:06,671 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/3f91a930ed884c919c81095ab3fa749c, entries=150, sequenceid=220, filesize=11.9 K 2024-11-25T03:39:06,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/be7b105c673a491ebdf838f86d655d5b as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/be7b105c673a491ebdf838f86d655d5b 2024-11-25T03:39:06,674 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/be7b105c673a491ebdf838f86d655d5b, entries=150, sequenceid=220, filesize=11.9 K 2024-11-25T03:39:06,675 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 502b5a21115d6420c81225a0df454848 in 1254ms, sequenceid=220, compaction requested=true 2024-11-25T03:39:06,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 502b5a21115d6420c81225a0df454848: 2024-11-25T03:39:06,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:06,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-25T03:39:06,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-25T03:39:06,679 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-25T03:39:06,679 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3970 sec 2024-11-25T03:39:06,681 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 3.4020 sec 2024-11-25T03:39:06,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 502b5a21115d6420c81225a0df454848 2024-11-25T03:39:06,746 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:39:06,746 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 502b5a21115d6420c81225a0df454848 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-25T03:39:06,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/8ad38a445e0e4bedaabdc006c66a0b3c is 50, key is test_row_0/A:col10/1732505946745/Put/seqid=0 2024-11-25T03:39:06,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742282_1458 (size=14541) 2024-11-25T03:39:06,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:06,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35728 deadline: 1732506006776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:06,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:06,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:06,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35738 deadline: 1732506006783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:06,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:06,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:06,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:06,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:06,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:06,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:06,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35728 deadline: 1732506006897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:06,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:06,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:06,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:06,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:06,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35738 deadline: 1732506006897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:06,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:06,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:06,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:07,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:07,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35728 deadline: 1732506007108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:07,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:07,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35738 deadline: 1732506007108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:07,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:07,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:07,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:07,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:07,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:07,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:07,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/8ad38a445e0e4bedaabdc006c66a0b3c 2024-11-25T03:39:07,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/562a67fd9ae64f8a866039d6f86e41ab is 50, key is test_row_0/B:col10/1732505946745/Put/seqid=0 2024-11-25T03:39:07,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742283_1459 (size=12151) 2024-11-25T03:39:07,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:07,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35728 deadline: 1732506007427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:07,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:07,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:07,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:07,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35738 deadline: 1732506007427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:07,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:07,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:07,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:07,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:07,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-25T03:39:07,437 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:39:07,438 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:39:07,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-25T03:39:07,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-25T03:39:07,439 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:39:07,440 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:39:07,440 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:39:07,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-25T03:39:07,566 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/562a67fd9ae64f8a866039d6f86e41ab 2024-11-25T03:39:07,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/c3b5ea28712f4d87be4fe397e072d935 is 50, key is test_row_0/C:col10/1732505946745/Put/seqid=0 2024-11-25T03:39:07,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742284_1460 (size=12151) 2024-11-25T03:39:07,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-25T03:39:07,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:07,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. as already flushing 2024-11-25T03:39:07,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:07,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:07,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:07,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:07,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-25T03:39:07,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:07,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. as already flushing 2024-11-25T03:39:07,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:07,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:07,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:07,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:07,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-25T03:39:07,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-25T03:39:07,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:07,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. as already flushing 2024-11-25T03:39:07,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:07,899 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:07,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:07,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:07,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:07,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35728 deadline: 1732506007947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:07,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:07,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:07,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:07,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35738 deadline: 1732506007947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:07,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:07,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:07,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:07,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:07,976 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/c3b5ea28712f4d87be4fe397e072d935 2024-11-25T03:39:07,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/8ad38a445e0e4bedaabdc006c66a0b3c as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/8ad38a445e0e4bedaabdc006c66a0b3c 2024-11-25T03:39:07,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/8ad38a445e0e4bedaabdc006c66a0b3c, entries=200, sequenceid=245, filesize=14.2 K 2024-11-25T03:39:07,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/562a67fd9ae64f8a866039d6f86e41ab as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/562a67fd9ae64f8a866039d6f86e41ab 2024-11-25T03:39:07,987 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/562a67fd9ae64f8a866039d6f86e41ab, entries=150, sequenceid=245, filesize=11.9 K 2024-11-25T03:39:07,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/c3b5ea28712f4d87be4fe397e072d935 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/c3b5ea28712f4d87be4fe397e072d935 2024-11-25T03:39:07,991 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/c3b5ea28712f4d87be4fe397e072d935, entries=150, sequenceid=245, filesize=11.9 K 2024-11-25T03:39:07,991 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 502b5a21115d6420c81225a0df454848 in 1245ms, sequenceid=245, compaction requested=true 2024-11-25T03:39:07,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 502b5a21115d6420c81225a0df454848: 2024-11-25T03:39:07,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 502b5a21115d6420c81225a0df454848:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:39:07,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:39:07,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 502b5a21115d6420c81225a0df454848:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:39:07,992 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:39:07,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:39:07,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 502b5a21115d6420c81225a0df454848:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:39:07,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T03:39:07,992 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:39:07,993 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:39:07,993 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53828 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:39:07,993 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): 502b5a21115d6420c81225a0df454848/A is initiating minor compaction (all files) 2024-11-25T03:39:07,993 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 502b5a21115d6420c81225a0df454848/B is initiating minor compaction (all files) 2024-11-25T03:39:07,993 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 502b5a21115d6420c81225a0df454848/B in TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:07,993 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 502b5a21115d6420c81225a0df454848/A in TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:07,993 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/a733e9b77e4c43a7a3aa97050c6a60c4, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/2f1dd915ee8948218fa833f21f299ee5, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/3f91a930ed884c919c81095ab3fa749c, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/562a67fd9ae64f8a866039d6f86e41ab] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp, totalSize=47.9 K 2024-11-25T03:39:07,993 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/8a5a815fbd5a4213a674e7667e51053e, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/18837405895c492d867015148033df07, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/308033b80bb54bfabcf73933613953db, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/8ad38a445e0e4bedaabdc006c66a0b3c] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp, totalSize=52.6 K 2024-11-25T03:39:07,993 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a5a815fbd5a4213a674e7667e51053e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732505941668 2024-11-25T03:39:07,993 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting a733e9b77e4c43a7a3aa97050c6a60c4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732505941668 2024-11-25T03:39:07,994 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18837405895c492d867015148033df07, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732505942922 2024-11-25T03:39:07,994 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f1dd915ee8948218fa833f21f299ee5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732505942922 2024-11-25T03:39:07,994 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 308033b80bb54bfabcf73933613953db, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732505944133 2024-11-25T03:39:07,994 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f91a930ed884c919c81095ab3fa749c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732505944133 2024-11-25T03:39:07,994 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ad38a445e0e4bedaabdc006c66a0b3c, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732505946402 2024-11-25T03:39:07,994 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 562a67fd9ae64f8a866039d6f86e41ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732505946402 2024-11-25T03:39:08,002 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 502b5a21115d6420c81225a0df454848#B#compaction#390 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:39:08,002 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 502b5a21115d6420c81225a0df454848#A#compaction#389 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:39:08,003 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/c013869156624e529db8f2f96e3329eb is 50, key is test_row_0/B:col10/1732505946745/Put/seqid=0 2024-11-25T03:39:08,003 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/109407ffb22e4b08a613169948ba6549 is 50, key is test_row_0/A:col10/1732505946745/Put/seqid=0 2024-11-25T03:39:08,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742285_1461 (size=12731) 2024-11-25T03:39:08,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742286_1462 (size=12731) 2024-11-25T03:39:08,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-25T03:39:08,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:08,052 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 502b5a21115d6420c81225a0df454848 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-25T03:39:08,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/e33981d1476240e98e719cd3588bfa6f is 50, key is test_row_0/A:col10/1732505946782/Put/seqid=0 2024-11-25T03:39:08,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742287_1463 (size=12151) 2024-11-25T03:39:08,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-25T03:39:08,410 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/c013869156624e529db8f2f96e3329eb as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/c013869156624e529db8f2f96e3329eb 2024-11-25T03:39:08,415 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/109407ffb22e4b08a613169948ba6549 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/109407ffb22e4b08a613169948ba6549 2024-11-25T03:39:08,416 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 502b5a21115d6420c81225a0df454848/B of 502b5a21115d6420c81225a0df454848 into c013869156624e529db8f2f96e3329eb(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:39:08,416 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 502b5a21115d6420c81225a0df454848: 2024-11-25T03:39:08,416 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., storeName=502b5a21115d6420c81225a0df454848/B, priority=12, startTime=1732505947992; duration=0sec 2024-11-25T03:39:08,416 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:39:08,416 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 502b5a21115d6420c81225a0df454848:B 2024-11-25T03:39:08,416 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:39:08,417 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:39:08,417 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 502b5a21115d6420c81225a0df454848/C is initiating minor compaction (all files) 2024-11-25T03:39:08,417 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 502b5a21115d6420c81225a0df454848/C in TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:08,418 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/d14f1d4953264608a69487c574258884, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/19ee9d62a219490ea319e6d843600cdd, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/be7b105c673a491ebdf838f86d655d5b, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/c3b5ea28712f4d87be4fe397e072d935] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp, totalSize=47.9 K 2024-11-25T03:39:08,418 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting d14f1d4953264608a69487c574258884, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732505941668 2024-11-25T03:39:08,418 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 19ee9d62a219490ea319e6d843600cdd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732505942922 2024-11-25T03:39:08,419 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting be7b105c673a491ebdf838f86d655d5b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732505944133 2024-11-25T03:39:08,420 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 502b5a21115d6420c81225a0df454848/A of 502b5a21115d6420c81225a0df454848 into 109407ffb22e4b08a613169948ba6549(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:39:08,420 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 502b5a21115d6420c81225a0df454848: 2024-11-25T03:39:08,420 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., storeName=502b5a21115d6420c81225a0df454848/A, priority=12, startTime=1732505947992; duration=0sec 2024-11-25T03:39:08,420 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:39:08,420 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 502b5a21115d6420c81225a0df454848:A 2024-11-25T03:39:08,420 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting c3b5ea28712f4d87be4fe397e072d935, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732505946402 2024-11-25T03:39:08,428 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 502b5a21115d6420c81225a0df454848#C#compaction#392 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:39:08,429 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/2a806caf9a22449babc793ca31651a2e is 50, key is test_row_0/C:col10/1732505946745/Put/seqid=0 2024-11-25T03:39:08,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742288_1464 (size=12731) 2024-11-25T03:39:08,460 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/e33981d1476240e98e719cd3588bfa6f 2024-11-25T03:39:08,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/1cb51df33573418e80c7b473896db18d is 50, key is test_row_0/B:col10/1732505946782/Put/seqid=0 2024-11-25T03:39:08,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742289_1465 (size=12151) 2024-11-25T03:39:08,481 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/1cb51df33573418e80c7b473896db18d 2024-11-25T03:39:08,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/2b6b429c48f24a1b9cbb989282a28cae is 50, key is test_row_0/C:col10/1732505946782/Put/seqid=0 2024-11-25T03:39:08,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742290_1466 (size=12151) 2024-11-25T03:39:08,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-25T03:39:08,837 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/2a806caf9a22449babc793ca31651a2e as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/2a806caf9a22449babc793ca31651a2e 2024-11-25T03:39:08,841 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 502b5a21115d6420c81225a0df454848/C of 502b5a21115d6420c81225a0df454848 into 2a806caf9a22449babc793ca31651a2e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:39:08,842 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 502b5a21115d6420c81225a0df454848: 2024-11-25T03:39:08,842 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., storeName=502b5a21115d6420c81225a0df454848/C, priority=12, startTime=1732505947992; duration=0sec 2024-11-25T03:39:08,842 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:39:08,842 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 502b5a21115d6420c81225a0df454848:C 2024-11-25T03:39:08,891 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/2b6b429c48f24a1b9cbb989282a28cae 2024-11-25T03:39:08,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/e33981d1476240e98e719cd3588bfa6f as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/e33981d1476240e98e719cd3588bfa6f 2024-11-25T03:39:08,899 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/e33981d1476240e98e719cd3588bfa6f, entries=150, sequenceid=256, filesize=11.9 K 2024-11-25T03:39:08,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/1cb51df33573418e80c7b473896db18d as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/1cb51df33573418e80c7b473896db18d 2024-11-25T03:39:08,903 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/1cb51df33573418e80c7b473896db18d, entries=150, sequenceid=256, filesize=11.9 K 2024-11-25T03:39:08,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/2b6b429c48f24a1b9cbb989282a28cae as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/2b6b429c48f24a1b9cbb989282a28cae 2024-11-25T03:39:08,906 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/2b6b429c48f24a1b9cbb989282a28cae, entries=150, sequenceid=256, filesize=11.9 K 2024-11-25T03:39:08,907 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=0 B/0 for 502b5a21115d6420c81225a0df454848 in 856ms, sequenceid=256, compaction requested=false 2024-11-25T03:39:08,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 502b5a21115d6420c81225a0df454848: 2024-11-25T03:39:08,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:08,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-25T03:39:08,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-25T03:39:08,911 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-25T03:39:08,911 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4690 sec 2024-11-25T03:39:08,913 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.4730 sec 2024-11-25T03:39:08,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 502b5a21115d6420c81225a0df454848 2024-11-25T03:39:08,973 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:39:08,973 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 502b5a21115d6420c81225a0df454848 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-25T03:39:08,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/2be075905ae044258b5b4d4c7b6a3a00 is 50, key is test_row_1/A:col10/1732505948958/Put/seqid=0 2024-11-25T03:39:08,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742291_1467 (size=12297) 2024-11-25T03:39:09,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:09,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35728 deadline: 1732506009041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:09,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:09,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:09,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:09,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:09,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35738 deadline: 1732506009049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:09,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:09,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:09,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:09,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:09,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35728 deadline: 1732506009157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:09,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:09,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:09,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:09,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:09,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35738 deadline: 1732506009167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:09,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:09,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:09,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:09,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:09,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1732506009357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:09,358 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,34881,1732505806588 for 'test_row_2' in 502b5a21115d6420c81225a0df454848 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:39:09,358 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:09,358 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:09,358 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:09,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:09,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35704 deadline: 1732506009367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:09,374 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,34881,1732505806588 for 'test_row_0' in 502b5a21115d6420c81225a0df454848 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:39:09,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:09,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:09,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:09,382 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/2be075905ae044258b5b4d4c7b6a3a00 2024-11-25T03:39:09,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:09,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35768 deadline: 1732506009377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:09,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35728 deadline: 1732506009377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:09,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:09,384 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,34881,1732505806588 for 'test_row_1' in 502b5a21115d6420c81225a0df454848 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8246 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:39:09,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:09,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:09,384 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:09,384 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:09,384 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:09,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/a2ff70455eb74ad2a7da681ecc1c6056 is 50, key is test_row_1/B:col10/1732505948958/Put/seqid=0 2024-11-25T03:39:09,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742292_1468 (size=9857) 2024-11-25T03:39:09,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:09,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35738 deadline: 1732506009387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:09,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:09,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:09,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:09,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-25T03:39:09,587 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:39:09,588 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:39:09,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-25T03:39:09,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-25T03:39:09,589 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:39:09,590 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:39:09,590 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:39:09,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:09,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35728 deadline: 1732506009688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:09,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:09,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:09,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:09,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-25T03:39:09,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:09,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35738 deadline: 1732506009697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:09,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:09,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:09,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:09,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-25T03:39:09,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:09,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. as already flushing 2024-11-25T03:39:09,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:09,742 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:09,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:09,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:09,792 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/a2ff70455eb74ad2a7da681ecc1c6056 2024-11-25T03:39:09,798 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/9c6b1f9fad1549ba92f283ca2c71542d is 50, key is test_row_1/C:col10/1732505948958/Put/seqid=0 2024-11-25T03:39:09,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742293_1469 (size=9857) 2024-11-25T03:39:09,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-25T03:39:09,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:09,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. as already flushing 2024-11-25T03:39:09,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:09,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:09,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:09,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:09,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-25T03:39:10,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-25T03:39:10,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:10,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. as already flushing 2024-11-25T03:39:10,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:10,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:10,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:10,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:10,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-25T03:39:10,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:10,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. as already flushing 2024-11-25T03:39:10,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:10,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:10,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:10,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:10,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/9c6b1f9fad1549ba92f283ca2c71542d 2024-11-25T03:39:10,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/2be075905ae044258b5b4d4c7b6a3a00 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/2be075905ae044258b5b4d4c7b6a3a00 2024-11-25T03:39:10,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:10,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35728 deadline: 1732506010208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:10,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:10,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:10,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:10,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/2be075905ae044258b5b4d4c7b6a3a00, entries=150, sequenceid=270, filesize=12.0 K 2024-11-25T03:39:10,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/a2ff70455eb74ad2a7da681ecc1c6056 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/a2ff70455eb74ad2a7da681ecc1c6056 2024-11-25T03:39:10,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-25T03:39:10,218 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/a2ff70455eb74ad2a7da681ecc1c6056, entries=100, sequenceid=270, filesize=9.6 K 2024-11-25T03:39:10,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/9c6b1f9fad1549ba92f283ca2c71542d as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/9c6b1f9fad1549ba92f283ca2c71542d 2024-11-25T03:39:10,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/9c6b1f9fad1549ba92f283ca2c71542d, entries=100, sequenceid=270, filesize=9.6 K 2024-11-25T03:39:10,223 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 502b5a21115d6420c81225a0df454848 in 1249ms, sequenceid=270, compaction requested=true 2024-11-25T03:39:10,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 502b5a21115d6420c81225a0df454848: 2024-11-25T03:39:10,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 502b5a21115d6420c81225a0df454848:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:39:10,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:39:10,223 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:39:10,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 502b5a21115d6420c81225a0df454848:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:39:10,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:39:10,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 502b5a21115d6420c81225a0df454848:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:39:10,223 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:39:10,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:39:10,224 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37179 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:39:10,224 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): 502b5a21115d6420c81225a0df454848/A is initiating minor compaction (all files) 2024-11-25T03:39:10,224 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 502b5a21115d6420c81225a0df454848/A in TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:10,224 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/109407ffb22e4b08a613169948ba6549, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/e33981d1476240e98e719cd3588bfa6f, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/2be075905ae044258b5b4d4c7b6a3a00] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp, totalSize=36.3 K 2024-11-25T03:39:10,224 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34739 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:39:10,225 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 109407ffb22e4b08a613169948ba6549, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732505946402 2024-11-25T03:39:10,225 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 502b5a21115d6420c81225a0df454848/B is initiating minor compaction (all files) 2024-11-25T03:39:10,225 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 502b5a21115d6420c81225a0df454848/B in TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:10,225 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/c013869156624e529db8f2f96e3329eb, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/1cb51df33573418e80c7b473896db18d, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/a2ff70455eb74ad2a7da681ecc1c6056] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp, totalSize=33.9 K 2024-11-25T03:39:10,225 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting c013869156624e529db8f2f96e3329eb, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732505946402 2024-11-25T03:39:10,225 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting e33981d1476240e98e719cd3588bfa6f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732505946755 2024-11-25T03:39:10,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 502b5a21115d6420c81225a0df454848 2024-11-25T03:39:10,225 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cb51df33573418e80c7b473896db18d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732505946755 2024-11-25T03:39:10,225 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:39:10,225 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2be075905ae044258b5b4d4c7b6a3a00, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732505948958 2024-11-25T03:39:10,225 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 502b5a21115d6420c81225a0df454848 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-25T03:39:10,226 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting a2ff70455eb74ad2a7da681ecc1c6056, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732505948958 2024-11-25T03:39:10,231 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 502b5a21115d6420c81225a0df454848#A#compaction#399 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:39:10,231 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/a0232b9f638f4f80b6c43f96a16be390 is 50, key is test_row_0/A:col10/1732505949017/Put/seqid=0 2024-11-25T03:39:10,231 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/6481cf9ef563460fa1415098134f4cae is 50, key is test_row_0/A:col10/1732505946782/Put/seqid=0 2024-11-25T03:39:10,234 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 502b5a21115d6420c81225a0df454848#B#compaction#400 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:39:10,235 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/766fedfc2bfe47eb97e732797aa655e6 is 50, key is test_row_0/B:col10/1732505946782/Put/seqid=0 2024-11-25T03:39:10,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742295_1471 (size=12933) 2024-11-25T03:39:10,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742294_1470 (size=14741) 2024-11-25T03:39:10,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742296_1472 (size=12933) 2024-11-25T03:39:10,251 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/766fedfc2bfe47eb97e732797aa655e6 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/766fedfc2bfe47eb97e732797aa655e6 2024-11-25T03:39:10,255 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 502b5a21115d6420c81225a0df454848/B of 502b5a21115d6420c81225a0df454848 into 766fedfc2bfe47eb97e732797aa655e6(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:39:10,255 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 502b5a21115d6420c81225a0df454848: 2024-11-25T03:39:10,255 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., storeName=502b5a21115d6420c81225a0df454848/B, priority=13, startTime=1732505950223; duration=0sec 2024-11-25T03:39:10,255 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:39:10,255 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 502b5a21115d6420c81225a0df454848:B 2024-11-25T03:39:10,255 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:39:10,256 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34739 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:39:10,256 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 502b5a21115d6420c81225a0df454848/C is initiating minor compaction (all files) 2024-11-25T03:39:10,256 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 502b5a21115d6420c81225a0df454848/C in TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:10,256 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/2a806caf9a22449babc793ca31651a2e, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/2b6b429c48f24a1b9cbb989282a28cae, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/9c6b1f9fad1549ba92f283ca2c71542d] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp, totalSize=33.9 K 2024-11-25T03:39:10,256 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a806caf9a22449babc793ca31651a2e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732505946402 2024-11-25T03:39:10,256 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b6b429c48f24a1b9cbb989282a28cae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732505946755 2024-11-25T03:39:10,256 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c6b1f9fad1549ba92f283ca2c71542d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732505948958 2024-11-25T03:39:10,263 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 502b5a21115d6420c81225a0df454848#C#compaction#401 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:39:10,263 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/244ba37e6e35482894ed28983f493aaf is 50, key is test_row_0/C:col10/1732505946782/Put/seqid=0 2024-11-25T03:39:10,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742297_1473 (size=12933) 2024-11-25T03:39:10,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:10,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35738 deadline: 1732506010284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:10,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:10,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:10,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:10,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-25T03:39:10,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:10,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. as already flushing 2024-11-25T03:39:10,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:10,354 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:10,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:10,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:10,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:10,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35738 deadline: 1732506010397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:10,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:10,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:10,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:10,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-25T03:39:10,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:10,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. as already flushing 2024-11-25T03:39:10,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:10,507 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:10,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:10,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:10,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:10,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35738 deadline: 1732506010617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:10,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:10,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:10,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:10,645 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/a0232b9f638f4f80b6c43f96a16be390 2024-11-25T03:39:10,646 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/6481cf9ef563460fa1415098134f4cae as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/6481cf9ef563460fa1415098134f4cae 2024-11-25T03:39:10,651 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 502b5a21115d6420c81225a0df454848/A of 502b5a21115d6420c81225a0df454848 into 6481cf9ef563460fa1415098134f4cae(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:39:10,651 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 502b5a21115d6420c81225a0df454848: 2024-11-25T03:39:10,651 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., storeName=502b5a21115d6420c81225a0df454848/A, priority=13, startTime=1732505950223; duration=0sec 2024-11-25T03:39:10,651 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:39:10,651 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 502b5a21115d6420c81225a0df454848:A 2024-11-25T03:39:10,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/ad87fea86a7d4861b999d985afe6d266 is 50, key is test_row_0/B:col10/1732505949017/Put/seqid=0 2024-11-25T03:39:10,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742298_1474 (size=12301) 2024-11-25T03:39:10,659 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/ad87fea86a7d4861b999d985afe6d266 2024-11-25T03:39:10,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-25T03:39:10,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:10,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. as already flushing 2024-11-25T03:39:10,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:10,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:10,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:10,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:10,668 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/58ea85975dad48308bc4a8a289055ab3 is 50, key is test_row_0/C:col10/1732505949017/Put/seqid=0 2024-11-25T03:39:10,671 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/244ba37e6e35482894ed28983f493aaf as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/244ba37e6e35482894ed28983f493aaf 2024-11-25T03:39:10,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742299_1475 (size=12301) 2024-11-25T03:39:10,672 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/58ea85975dad48308bc4a8a289055ab3 2024-11-25T03:39:10,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/a0232b9f638f4f80b6c43f96a16be390 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/a0232b9f638f4f80b6c43f96a16be390 2024-11-25T03:39:10,677 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 502b5a21115d6420c81225a0df454848/C of 502b5a21115d6420c81225a0df454848 into 244ba37e6e35482894ed28983f493aaf(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:39:10,677 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 502b5a21115d6420c81225a0df454848: 2024-11-25T03:39:10,677 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., storeName=502b5a21115d6420c81225a0df454848/C, priority=13, startTime=1732505950223; duration=0sec 2024-11-25T03:39:10,677 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:39:10,677 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 502b5a21115d6420c81225a0df454848:C 2024-11-25T03:39:10,680 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/a0232b9f638f4f80b6c43f96a16be390, entries=200, sequenceid=296, filesize=14.4 K 2024-11-25T03:39:10,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/ad87fea86a7d4861b999d985afe6d266 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/ad87fea86a7d4861b999d985afe6d266 2024-11-25T03:39:10,683 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/ad87fea86a7d4861b999d985afe6d266, entries=150, sequenceid=296, filesize=12.0 K 2024-11-25T03:39:10,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/58ea85975dad48308bc4a8a289055ab3 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/58ea85975dad48308bc4a8a289055ab3 2024-11-25T03:39:10,687 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/58ea85975dad48308bc4a8a289055ab3, entries=150, sequenceid=296, filesize=12.0 K 2024-11-25T03:39:10,687 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 502b5a21115d6420c81225a0df454848 in 462ms, sequenceid=296, compaction requested=false 2024-11-25T03:39:10,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 502b5a21115d6420c81225a0df454848: 2024-11-25T03:39:10,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-25T03:39:10,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-25T03:39:10,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:10,812 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 502b5a21115d6420c81225a0df454848 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-25T03:39:10,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/bc964d8e432143af89f5b97489d20b94 is 50, key is test_row_0/A:col10/1732505950268/Put/seqid=0 2024-11-25T03:39:10,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742300_1476 (size=12301) 2024-11-25T03:39:10,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 502b5a21115d6420c81225a0df454848 2024-11-25T03:39:10,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. as already flushing 2024-11-25T03:39:11,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:11,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35738 deadline: 1732506011143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:11,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:11,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:11,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:11,221 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/bc964d8e432143af89f5b97489d20b94 2024-11-25T03:39:11,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/641148dc4bf74a11b75f54998fee8e5e is 50, key is test_row_0/B:col10/1732505950268/Put/seqid=0 2024-11-25T03:39:11,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742301_1477 (size=12301) 2024-11-25T03:39:11,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:11,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35728 deadline: 1732506011228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:11,237 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,34881,1732505806588 for 'test_row_1' in 502b5a21115d6420c81225a0df454848 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:39:11,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:11,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:11,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:11,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:11,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35738 deadline: 1732506011257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:11,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:11,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:11,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:11,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:11,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35738 deadline: 1732506011468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:11,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:11,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:11,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:11,632 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/641148dc4bf74a11b75f54998fee8e5e 2024-11-25T03:39:11,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/3d7bfbeca6b3416d83ea189100233638 is 50, key is test_row_0/C:col10/1732505950268/Put/seqid=0 2024-11-25T03:39:11,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742302_1478 (size=12301) 2024-11-25T03:39:11,643 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/3d7bfbeca6b3416d83ea189100233638 2024-11-25T03:39:11,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/bc964d8e432143af89f5b97489d20b94 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/bc964d8e432143af89f5b97489d20b94 2024-11-25T03:39:11,649 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/bc964d8e432143af89f5b97489d20b94, entries=150, sequenceid=309, filesize=12.0 K 2024-11-25T03:39:11,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/641148dc4bf74a11b75f54998fee8e5e as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/641148dc4bf74a11b75f54998fee8e5e 2024-11-25T03:39:11,652 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/641148dc4bf74a11b75f54998fee8e5e, entries=150, sequenceid=309, filesize=12.0 K 2024-11-25T03:39:11,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/3d7bfbeca6b3416d83ea189100233638 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/3d7bfbeca6b3416d83ea189100233638 2024-11-25T03:39:11,656 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/3d7bfbeca6b3416d83ea189100233638, entries=150, sequenceid=309, filesize=12.0 K 2024-11-25T03:39:11,657 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 502b5a21115d6420c81225a0df454848 in 845ms, sequenceid=309, compaction requested=true 2024-11-25T03:39:11,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 502b5a21115d6420c81225a0df454848: 2024-11-25T03:39:11,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:11,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-25T03:39:11,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-25T03:39:11,659 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-25T03:39:11,660 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0680 sec 2024-11-25T03:39:11,661 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 2.0720 sec 2024-11-25T03:39:11,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-25T03:39:11,748 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:39:11,748 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:39:11,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-25T03:39:11,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-25T03:39:11,750 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:39:11,751 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:39:11,751 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:39:11,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 502b5a21115d6420c81225a0df454848 2024-11-25T03:39:11,795 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:39:11,795 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 502b5a21115d6420c81225a0df454848 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-25T03:39:11,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/8eb22a1cfb004bd98a3c38fe23ea6791 is 50, key is test_row_0/A:col10/1732505951794/Put/seqid=0 2024-11-25T03:39:11,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742303_1479 (size=14741) 2024-11-25T03:39:11,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:11,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35738 deadline: 1732506011837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:11,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:11,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:11,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:11,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-25T03:39:11,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-25T03:39:11,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:11,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. as already flushing 2024-11-25T03:39:11,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:11,904 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:11,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:11,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:11,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:11,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35738 deadline: 1732506011957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:11,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:11,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:11,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:12,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-25T03:39:12,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:12,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. as already flushing 2024-11-25T03:39:12,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:12,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:12,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:12,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:12,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-25T03:39:12,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:12,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35738 deadline: 1732506012168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:12,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:12,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:12,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:12,203 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/8eb22a1cfb004bd98a3c38fe23ea6791 2024-11-25T03:39:12,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-25T03:39:12,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:12,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. as already flushing 2024-11-25T03:39:12,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:12,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:12,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:12,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:12,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/e459daf8ce8048e2abf9482c88db8f47 is 50, key is test_row_0/B:col10/1732505951794/Put/seqid=0 2024-11-25T03:39:12,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742304_1480 (size=12301) 2024-11-25T03:39:12,215 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/e459daf8ce8048e2abf9482c88db8f47 2024-11-25T03:39:12,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/e2bf703df961419f9db50244019dab35 is 50, key is test_row_0/C:col10/1732505951794/Put/seqid=0 2024-11-25T03:39:12,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742305_1481 (size=12301) 2024-11-25T03:39:12,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-25T03:39:12,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:12,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. as already flushing 2024-11-25T03:39:12,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:12,362 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:12,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:12,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:12,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-25T03:39:12,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:12,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35738 deadline: 1732506012477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:12,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:12,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:12,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:12,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-25T03:39:12,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:12,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. as already flushing 2024-11-25T03:39:12,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:12,515 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:12,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:12,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:12,626 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/e2bf703df961419f9db50244019dab35 2024-11-25T03:39:12,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/8eb22a1cfb004bd98a3c38fe23ea6791 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/8eb22a1cfb004bd98a3c38fe23ea6791 2024-11-25T03:39:12,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/8eb22a1cfb004bd98a3c38fe23ea6791, entries=200, sequenceid=336, filesize=14.4 K 2024-11-25T03:39:12,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/e459daf8ce8048e2abf9482c88db8f47 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/e459daf8ce8048e2abf9482c88db8f47 2024-11-25T03:39:12,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/e459daf8ce8048e2abf9482c88db8f47, entries=150, sequenceid=336, filesize=12.0 K 2024-11-25T03:39:12,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/e2bf703df961419f9db50244019dab35 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/e2bf703df961419f9db50244019dab35 2024-11-25T03:39:12,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/e2bf703df961419f9db50244019dab35, entries=150, sequenceid=336, filesize=12.0 K 2024-11-25T03:39:12,641 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 502b5a21115d6420c81225a0df454848 in 845ms, sequenceid=336, compaction requested=true 2024-11-25T03:39:12,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 502b5a21115d6420c81225a0df454848: 2024-11-25T03:39:12,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 502b5a21115d6420c81225a0df454848:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:39:12,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:39:12,641 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:39:12,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 502b5a21115d6420c81225a0df454848:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:39:12,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:39:12,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 502b5a21115d6420c81225a0df454848:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:39:12,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:39:12,641 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:39:12,642 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54716 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:39:12,642 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49836 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:39:12,642 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): 502b5a21115d6420c81225a0df454848/A is initiating minor compaction (all files) 2024-11-25T03:39:12,642 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 502b5a21115d6420c81225a0df454848/B is initiating minor compaction (all files) 2024-11-25T03:39:12,642 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 502b5a21115d6420c81225a0df454848/B in TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:12,642 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 502b5a21115d6420c81225a0df454848/A in TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:12,642 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/766fedfc2bfe47eb97e732797aa655e6, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/ad87fea86a7d4861b999d985afe6d266, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/641148dc4bf74a11b75f54998fee8e5e, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/e459daf8ce8048e2abf9482c88db8f47] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp, totalSize=48.7 K 2024-11-25T03:39:12,642 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/6481cf9ef563460fa1415098134f4cae, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/a0232b9f638f4f80b6c43f96a16be390, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/bc964d8e432143af89f5b97489d20b94, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/8eb22a1cfb004bd98a3c38fe23ea6791] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp, totalSize=53.4 K 2024-11-25T03:39:12,643 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 766fedfc2bfe47eb97e732797aa655e6, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732505946782 2024-11-25T03:39:12,643 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6481cf9ef563460fa1415098134f4cae, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732505946782 2024-11-25T03:39:12,643 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0232b9f638f4f80b6c43f96a16be390, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1732505949017 2024-11-25T03:39:12,643 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting ad87fea86a7d4861b999d985afe6d266, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1732505949017 2024-11-25T03:39:12,643 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc964d8e432143af89f5b97489d20b94, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732505950235 2024-11-25T03:39:12,643 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 641148dc4bf74a11b75f54998fee8e5e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732505950235 2024-11-25T03:39:12,644 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8eb22a1cfb004bd98a3c38fe23ea6791, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732505951107 2024-11-25T03:39:12,644 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting e459daf8ce8048e2abf9482c88db8f47, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732505951128 2024-11-25T03:39:12,652 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 502b5a21115d6420c81225a0df454848#B#compaction#410 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:39:12,652 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 502b5a21115d6420c81225a0df454848#A#compaction#411 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:39:12,653 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/9d6e0b52dfca43c9840b00ac77cd1183 is 50, key is test_row_0/A:col10/1732505951794/Put/seqid=0 2024-11-25T03:39:12,653 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/fa2b57e788c14e92bf4f0095b022405e is 50, key is test_row_0/B:col10/1732505951794/Put/seqid=0 2024-11-25T03:39:12,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742306_1482 (size=13119) 2024-11-25T03:39:12,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-25T03:39:12,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:12,668 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 502b5a21115d6420c81225a0df454848 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-25T03:39:12,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742307_1483 (size=13119) 2024-11-25T03:39:12,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/0a2b784783a4427facabd199889863e3 is 50, key is test_row_0/A:col10/1732505951805/Put/seqid=0 2024-11-25T03:39:12,675 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/fa2b57e788c14e92bf4f0095b022405e as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/fa2b57e788c14e92bf4f0095b022405e 2024-11-25T03:39:12,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742308_1484 (size=12301) 2024-11-25T03:39:12,679 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 502b5a21115d6420c81225a0df454848/B of 502b5a21115d6420c81225a0df454848 into fa2b57e788c14e92bf4f0095b022405e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:39:12,679 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 502b5a21115d6420c81225a0df454848: 2024-11-25T03:39:12,679 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., storeName=502b5a21115d6420c81225a0df454848/B, priority=12, startTime=1732505952641; duration=0sec 2024-11-25T03:39:12,680 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:39:12,680 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 502b5a21115d6420c81225a0df454848:B 2024-11-25T03:39:12,680 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:39:12,681 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49836 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:39:12,681 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 502b5a21115d6420c81225a0df454848/C is initiating minor compaction (all files) 2024-11-25T03:39:12,681 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 502b5a21115d6420c81225a0df454848/C in TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:12,681 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/244ba37e6e35482894ed28983f493aaf, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/58ea85975dad48308bc4a8a289055ab3, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/3d7bfbeca6b3416d83ea189100233638, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/e2bf703df961419f9db50244019dab35] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp, totalSize=48.7 K 2024-11-25T03:39:12,681 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 244ba37e6e35482894ed28983f493aaf, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732505946782 2024-11-25T03:39:12,682 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 58ea85975dad48308bc4a8a289055ab3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1732505949017 2024-11-25T03:39:12,682 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d7bfbeca6b3416d83ea189100233638, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732505950235 2024-11-25T03:39:12,682 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting e2bf703df961419f9db50244019dab35, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732505951128 2024-11-25T03:39:12,691 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 502b5a21115d6420c81225a0df454848#C#compaction#413 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:39:12,691 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/18fe05b806984002a2bae50a1f1bf049 is 50, key is test_row_0/C:col10/1732505951794/Put/seqid=0 2024-11-25T03:39:12,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742309_1485 (size=13119) 2024-11-25T03:39:12,706 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/18fe05b806984002a2bae50a1f1bf049 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/18fe05b806984002a2bae50a1f1bf049 2024-11-25T03:39:12,711 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 502b5a21115d6420c81225a0df454848/C of 502b5a21115d6420c81225a0df454848 into 18fe05b806984002a2bae50a1f1bf049(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:39:12,711 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 502b5a21115d6420c81225a0df454848: 2024-11-25T03:39:12,711 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., storeName=502b5a21115d6420c81225a0df454848/C, priority=12, startTime=1732505952641; duration=0sec 2024-11-25T03:39:12,711 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:39:12,711 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 502b5a21115d6420c81225a0df454848:C 2024-11-25T03:39:12,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-25T03:39:13,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 502b5a21115d6420c81225a0df454848 2024-11-25T03:39:13,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. as already flushing 2024-11-25T03:39:13,067 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/9d6e0b52dfca43c9840b00ac77cd1183 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/9d6e0b52dfca43c9840b00ac77cd1183 2024-11-25T03:39:13,071 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 502b5a21115d6420c81225a0df454848/A of 502b5a21115d6420c81225a0df454848 into 9d6e0b52dfca43c9840b00ac77cd1183(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:39:13,071 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 502b5a21115d6420c81225a0df454848: 2024-11-25T03:39:13,071 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., storeName=502b5a21115d6420c81225a0df454848/A, priority=12, startTime=1732505952641; duration=0sec 2024-11-25T03:39:13,071 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:39:13,071 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 502b5a21115d6420c81225a0df454848:A 2024-11-25T03:39:13,079 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/0a2b784783a4427facabd199889863e3 2024-11-25T03:39:13,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/f8af0af7e9394887a698b795efec92b7 is 50, key is test_row_0/B:col10/1732505951805/Put/seqid=0 2024-11-25T03:39:13,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742310_1486 (size=12301) 2024-11-25T03:39:13,090 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/f8af0af7e9394887a698b795efec92b7 2024-11-25T03:39:13,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/58b3ae52cbf94bd3b7b5d91fd048397d is 50, key is test_row_0/C:col10/1732505951805/Put/seqid=0 2024-11-25T03:39:13,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742311_1487 (size=12301) 2024-11-25T03:39:13,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:13,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35738 deadline: 1732506013190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:13,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:13,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:13,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:13,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:13,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35728 deadline: 1732506013248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:13,258 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,34881,1732505806588 for 'test_row_1' in 502b5a21115d6420c81225a0df454848 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:39:13,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:13,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:13,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:13,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:13,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35738 deadline: 1732506013307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:13,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 , the old value is region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:13,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:13,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., hostname=38a5d2077268,34881,1732505806588, seqNum=2 because the exception is null or not the one we care about 2024-11-25T03:39:13,347 INFO [Thread-1872 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1872. 2024-11-25T03:39:13,347 DEBUG [Thread-1872 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:39:13,347 DEBUG [Thread-1872 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:13,347 DEBUG [Thread-1872 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:13,347 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:39:13,350 INFO [Thread-1870 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1870. 2024-11-25T03:39:13,350 DEBUG [Thread-1870 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:39:13,350 DEBUG [Thread-1870 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:13,350 DEBUG [Thread-1870 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:13,350 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:39:13,353 INFO [Thread-1868 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1868. 2024-11-25T03:39:13,353 DEBUG [Thread-1868 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:39:13,353 DEBUG [Thread-1868 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:13,353 DEBUG [Thread-1868 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:13,353 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:39:13,357 INFO [Thread-1876 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1876. 2024-11-25T03:39:13,357 DEBUG [Thread-1876 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:39:13,357 DEBUG [Thread-1876 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:13,357 DEBUG [Thread-1876 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:13,357 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:39:13,359 INFO [Thread-1874 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1874. 2024-11-25T03:39:13,359 DEBUG [Thread-1874 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:39:13,359 DEBUG [Thread-1874 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:13,359 DEBUG [Thread-1874 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:13,359 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:39:13,500 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/58b3ae52cbf94bd3b7b5d91fd048397d 2024-11-25T03:39:13,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/0a2b784783a4427facabd199889863e3 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/0a2b784783a4427facabd199889863e3 2024-11-25T03:39:13,507 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/0a2b784783a4427facabd199889863e3, entries=150, sequenceid=345, filesize=12.0 K 2024-11-25T03:39:13,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/f8af0af7e9394887a698b795efec92b7 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/f8af0af7e9394887a698b795efec92b7 2024-11-25T03:39:13,511 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/f8af0af7e9394887a698b795efec92b7, entries=150, sequenceid=345, filesize=12.0 K 2024-11-25T03:39:13,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/58b3ae52cbf94bd3b7b5d91fd048397d as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/58b3ae52cbf94bd3b7b5d91fd048397d 2024-11-25T03:39:13,515 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/58b3ae52cbf94bd3b7b5d91fd048397d, entries=150, sequenceid=345, filesize=12.0 K 2024-11-25T03:39:13,515 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 502b5a21115d6420c81225a0df454848 in 847ms, sequenceid=345, compaction requested=false 2024-11-25T03:39:13,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 502b5a21115d6420c81225a0df454848: 2024-11-25T03:39:13,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:13,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-25T03:39:13,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-25T03:39:13,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 502b5a21115d6420c81225a0df454848 2024-11-25T03:39:13,518 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:39:13,518 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 502b5a21115d6420c81225a0df454848 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-25T03:39:13,518 INFO [Thread-1861 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1861. 2024-11-25T03:39:13,518 DEBUG [Thread-1861 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:39:13,518 DEBUG [Thread-1861 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:13,518 DEBUG [Thread-1861 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:13,518 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:39:13,518 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-25T03:39:13,518 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7660 sec 2024-11-25T03:39:13,520 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.7710 sec 2024-11-25T03:39:13,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/d3f06f8b5d3840a99eeacde3f24d8655 is 50, key is test_row_0/A:col10/1732505953517/Put/seqid=0 2024-11-25T03:39:13,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742312_1488 (size=12301) 2024-11-25T03:39:13,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-25T03:39:13,898 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:39:13,925 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/d3f06f8b5d3840a99eeacde3f24d8655 2024-11-25T03:39:13,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/7c793734468d4063bd3be7568f72fb9a is 50, key is test_row_0/B:col10/1732505953517/Put/seqid=0 2024-11-25T03:39:13,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742313_1489 (size=12301) 2024-11-25T03:39:14,334 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/7c793734468d4063bd3be7568f72fb9a 2024-11-25T03:39:14,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/a32c97aadc954311a675467662e5d7cd is 50, key is test_row_0/C:col10/1732505953517/Put/seqid=0 2024-11-25T03:39:14,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742314_1490 (size=12301) 2024-11-25T03:39:14,743 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/a32c97aadc954311a675467662e5d7cd 2024-11-25T03:39:14,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/d3f06f8b5d3840a99eeacde3f24d8655 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/d3f06f8b5d3840a99eeacde3f24d8655 2024-11-25T03:39:14,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/d3f06f8b5d3840a99eeacde3f24d8655, entries=150, sequenceid=376, filesize=12.0 K 2024-11-25T03:39:14,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/7c793734468d4063bd3be7568f72fb9a as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/7c793734468d4063bd3be7568f72fb9a 2024-11-25T03:39:14,752 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/7c793734468d4063bd3be7568f72fb9a, entries=150, sequenceid=376, filesize=12.0 K 2024-11-25T03:39:14,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/a32c97aadc954311a675467662e5d7cd as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/a32c97aadc954311a675467662e5d7cd 2024-11-25T03:39:14,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/a32c97aadc954311a675467662e5d7cd, entries=150, sequenceid=376, filesize=12.0 K 2024-11-25T03:39:14,755 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=0 B/0 for 502b5a21115d6420c81225a0df454848 in 1237ms, sequenceid=376, compaction requested=true 2024-11-25T03:39:14,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 502b5a21115d6420c81225a0df454848: 2024-11-25T03:39:14,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 502b5a21115d6420c81225a0df454848:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:39:14,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:39:14,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 502b5a21115d6420c81225a0df454848:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:39:14,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:39:14,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 502b5a21115d6420c81225a0df454848:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:39:14,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:39:14,755 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:39:14,755 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:39:14,756 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:39:14,756 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:39:14,756 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 502b5a21115d6420c81225a0df454848/B is initiating minor compaction (all files) 2024-11-25T03:39:14,756 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): 502b5a21115d6420c81225a0df454848/A is initiating minor compaction (all files) 2024-11-25T03:39:14,756 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 502b5a21115d6420c81225a0df454848/B in TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:14,756 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 502b5a21115d6420c81225a0df454848/A in TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:14,756 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/fa2b57e788c14e92bf4f0095b022405e, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/f8af0af7e9394887a698b795efec92b7, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/7c793734468d4063bd3be7568f72fb9a] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp, totalSize=36.8 K 2024-11-25T03:39:14,756 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/9d6e0b52dfca43c9840b00ac77cd1183, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/0a2b784783a4427facabd199889863e3, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/d3f06f8b5d3840a99eeacde3f24d8655] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp, totalSize=36.8 K 2024-11-25T03:39:14,756 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting fa2b57e788c14e92bf4f0095b022405e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732505951128 2024-11-25T03:39:14,756 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d6e0b52dfca43c9840b00ac77cd1183, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732505951128 2024-11-25T03:39:14,757 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting f8af0af7e9394887a698b795efec92b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732505951805 2024-11-25T03:39:14,757 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a2b784783a4427facabd199889863e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732505951805 2024-11-25T03:39:14,757 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c793734468d4063bd3be7568f72fb9a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1732505953174 2024-11-25T03:39:14,757 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3f06f8b5d3840a99eeacde3f24d8655, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1732505953174 2024-11-25T03:39:14,763 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 502b5a21115d6420c81225a0df454848#B#compaction#419 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:39:14,763 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 502b5a21115d6420c81225a0df454848#A#compaction#420 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:39:14,763 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/e4614e329eb84b36a12bdbd8afe76f17 is 50, key is test_row_0/B:col10/1732505953517/Put/seqid=0 2024-11-25T03:39:14,763 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/94fd374a1e9d454bb39d11f2847ff19d is 50, key is test_row_0/A:col10/1732505953517/Put/seqid=0 2024-11-25T03:39:14,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742315_1491 (size=13221) 2024-11-25T03:39:14,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742316_1492 (size=13221) 2024-11-25T03:39:14,799 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T03:39:15,172 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/e4614e329eb84b36a12bdbd8afe76f17 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/e4614e329eb84b36a12bdbd8afe76f17 2024-11-25T03:39:15,172 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/94fd374a1e9d454bb39d11f2847ff19d as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/94fd374a1e9d454bb39d11f2847ff19d 2024-11-25T03:39:15,176 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 502b5a21115d6420c81225a0df454848/B of 502b5a21115d6420c81225a0df454848 into e4614e329eb84b36a12bdbd8afe76f17(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:39:15,176 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 502b5a21115d6420c81225a0df454848/A of 502b5a21115d6420c81225a0df454848 into 94fd374a1e9d454bb39d11f2847ff19d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:39:15,176 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 502b5a21115d6420c81225a0df454848: 2024-11-25T03:39:15,176 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 502b5a21115d6420c81225a0df454848: 2024-11-25T03:39:15,176 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., storeName=502b5a21115d6420c81225a0df454848/B, priority=13, startTime=1732505954755; duration=0sec 2024-11-25T03:39:15,176 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., storeName=502b5a21115d6420c81225a0df454848/A, priority=13, startTime=1732505954755; duration=0sec 2024-11-25T03:39:15,176 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:39:15,176 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:39:15,176 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 502b5a21115d6420c81225a0df454848:A 2024-11-25T03:39:15,176 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 502b5a21115d6420c81225a0df454848:B 2024-11-25T03:39:15,176 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:39:15,177 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:39:15,177 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): 502b5a21115d6420c81225a0df454848/C is initiating minor compaction (all files) 2024-11-25T03:39:15,177 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 502b5a21115d6420c81225a0df454848/C in TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:15,177 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/18fe05b806984002a2bae50a1f1bf049, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/58b3ae52cbf94bd3b7b5d91fd048397d, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/a32c97aadc954311a675467662e5d7cd] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp, totalSize=36.8 K 2024-11-25T03:39:15,177 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18fe05b806984002a2bae50a1f1bf049, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732505951128 2024-11-25T03:39:15,177 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58b3ae52cbf94bd3b7b5d91fd048397d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732505951805 2024-11-25T03:39:15,178 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting a32c97aadc954311a675467662e5d7cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1732505953174 2024-11-25T03:39:15,183 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 502b5a21115d6420c81225a0df454848#C#compaction#421 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:39:15,183 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/ea396025d4184fe888fca491aaffa996 is 50, key is test_row_0/C:col10/1732505953517/Put/seqid=0 2024-11-25T03:39:15,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742317_1493 (size=13221) 2024-11-25T03:39:15,590 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/ea396025d4184fe888fca491aaffa996 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/ea396025d4184fe888fca491aaffa996 2024-11-25T03:39:15,593 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 502b5a21115d6420c81225a0df454848/C of 502b5a21115d6420c81225a0df454848 into ea396025d4184fe888fca491aaffa996(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:39:15,593 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 502b5a21115d6420c81225a0df454848: 2024-11-25T03:39:15,593 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., storeName=502b5a21115d6420c81225a0df454848/C, priority=13, startTime=1732505954755; duration=0sec 2024-11-25T03:39:15,593 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:39:15,593 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 502b5a21115d6420c81225a0df454848:C 2024-11-25T03:39:17,298 INFO [Thread-1863 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1863. 2024-11-25T03:39:17,298 DEBUG [Thread-1863 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:39:17,298 DEBUG [Thread-1863 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:17,298 DEBUG [Thread-1863 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:17,298 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:39:19,398 INFO [Thread-1859 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1859. 2024-11-25T03:39:19,398 DEBUG [Thread-1859 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:39:19,398 DEBUG [Thread-1859 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:19,398 DEBUG [Thread-1859 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:19,398 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:39:19,428 INFO [Thread-1865 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1865. 2024-11-25T03:39:19,428 DEBUG [Thread-1865 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:39:19,428 DEBUG [Thread-1865 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:19,428 DEBUG [Thread-1865 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:19,428 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:39:19,479 INFO [Thread-1857 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1857. 2024-11-25T03:39:19,479 DEBUG [Thread-1857 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:39:19,479 DEBUG [Thread-1857 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:19,479 DEBUG [Thread-1857 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:19,479 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:39:19,479 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-25T03:39:19,479 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-11-25T03:39:19,479 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 32 2024-11-25T03:39:19,479 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 113 2024-11-25T03:39:19,479 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-25T03:39:19,479 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-25T03:39:19,479 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-25T03:39:19,479 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-25T03:39:19,479 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1463 2024-11-25T03:39:19,479 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4388 rows 2024-11-25T03:39:19,479 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1465 2024-11-25T03:39:19,479 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4390 rows 2024-11-25T03:39:19,479 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1467 2024-11-25T03:39:19,479 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4400 rows 2024-11-25T03:39:19,479 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1460 2024-11-25T03:39:19,479 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4379 rows 2024-11-25T03:39:19,479 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1467 2024-11-25T03:39:19,479 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4400 rows 2024-11-25T03:39:19,479 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T03:39:19,479 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T03:39:19,479 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:19,480 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:19,480 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:39:19,480 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-25T03:39:19,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-25T03:39:19,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-25T03:39:19,482 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732505959482"}]},"ts":"1732505959482"} 2024-11-25T03:39:19,483 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-25T03:39:19,483 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-25T03:39:19,484 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T03:39:19,485 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=502b5a21115d6420c81225a0df454848, UNASSIGN}] 2024-11-25T03:39:19,486 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=502b5a21115d6420c81225a0df454848, UNASSIGN 2024-11-25T03:39:19,486 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=502b5a21115d6420c81225a0df454848, regionState=CLOSING, regionLocation=38a5d2077268,34881,1732505806588 2024-11-25T03:39:19,487 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=502b5a21115d6420c81225a0df454848, UNASSIGN because future has completed 2024-11-25T03:39:19,487 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T03:39:19,487 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure 502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588}] 2024-11-25T03:39:19,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-25T03:39:19,639 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(122): Close 502b5a21115d6420c81225a0df454848 2024-11-25T03:39:19,639 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T03:39:19,639 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1722): Closing 502b5a21115d6420c81225a0df454848, disabling compactions & flushes 2024-11-25T03:39:19,639 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:19,639 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:19,639 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. after waiting 0 ms 2024-11-25T03:39:19,639 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:19,640 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(2902): Flushing 502b5a21115d6420c81225a0df454848 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-25T03:39:19,643 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/bd6e2141e8fe46b8b4c1b2ee214ef0d0 is 50, key is test_row_0/A:col10/1732505959478/Put/seqid=0 2024-11-25T03:39:19,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742318_1494 (size=12301) 2024-11-25T03:39:19,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-25T03:39:20,046 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/bd6e2141e8fe46b8b4c1b2ee214ef0d0 2024-11-25T03:39:20,052 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/8adfa28bc7fd42c9bf056a318e40a4b4 is 50, key is test_row_0/B:col10/1732505959478/Put/seqid=0 2024-11-25T03:39:20,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742319_1495 (size=12301) 2024-11-25T03:39:20,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-25T03:39:20,455 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/8adfa28bc7fd42c9bf056a318e40a4b4 2024-11-25T03:39:20,460 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/352a2af853664b689edff6e3f547f381 is 50, key is test_row_0/C:col10/1732505959478/Put/seqid=0 2024-11-25T03:39:20,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742320_1496 (size=12301) 2024-11-25T03:39:20,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-25T03:39:20,864 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/352a2af853664b689edff6e3f547f381 2024-11-25T03:39:20,867 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/A/bd6e2141e8fe46b8b4c1b2ee214ef0d0 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/bd6e2141e8fe46b8b4c1b2ee214ef0d0 2024-11-25T03:39:20,870 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/bd6e2141e8fe46b8b4c1b2ee214ef0d0, entries=150, sequenceid=386, filesize=12.0 K 2024-11-25T03:39:20,870 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/B/8adfa28bc7fd42c9bf056a318e40a4b4 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/8adfa28bc7fd42c9bf056a318e40a4b4 2024-11-25T03:39:20,873 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/8adfa28bc7fd42c9bf056a318e40a4b4, entries=150, sequenceid=386, filesize=12.0 K 2024-11-25T03:39:20,873 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/.tmp/C/352a2af853664b689edff6e3f547f381 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/352a2af853664b689edff6e3f547f381 2024-11-25T03:39:20,876 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/352a2af853664b689edff6e3f547f381, entries=150, sequenceid=386, filesize=12.0 K 2024-11-25T03:39:20,876 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 502b5a21115d6420c81225a0df454848 in 1237ms, sequenceid=386, compaction requested=false 2024-11-25T03:39:20,877 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/867146695ed545839e8e4005e179805a, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/10c789d1aebb4595992fcbd7d141f22c, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/24f5bae460c2407a8ba9d16d45ea22c4, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/c6b6f01ded5f4c60aa7d4093e19c7d51, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/2b66e9b80eb643cc9809dfca1de47920, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/d20c677351d84b00888e0da1555e7a40, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/7d7933614a8c47449bb3a2fe135c3738, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/84624cbac1ed4ebfa508bf04846a1454, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/9e6dfc9223da4882989e4786eb5ae2ee, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/9ffe5326d6ce4c4eb668787eccd572a4, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/fdcb53913c32436d9c3c2bfbc407bb7d, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/674a6f804c9f4f2a8051bb20e920376f, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/74d25f312513457a9080b395614dffe7, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/8a5a815fbd5a4213a674e7667e51053e, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/18837405895c492d867015148033df07, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/308033b80bb54bfabcf73933613953db, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/8ad38a445e0e4bedaabdc006c66a0b3c, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/109407ffb22e4b08a613169948ba6549, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/e33981d1476240e98e719cd3588bfa6f, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/6481cf9ef563460fa1415098134f4cae, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/2be075905ae044258b5b4d4c7b6a3a00, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/a0232b9f638f4f80b6c43f96a16be390, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/bc964d8e432143af89f5b97489d20b94, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/8eb22a1cfb004bd98a3c38fe23ea6791, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/9d6e0b52dfca43c9840b00ac77cd1183, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/0a2b784783a4427facabd199889863e3, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/d3f06f8b5d3840a99eeacde3f24d8655] to archive 2024-11-25T03:39:20,878 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T03:39:20,879 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/867146695ed545839e8e4005e179805a to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/867146695ed545839e8e4005e179805a 2024-11-25T03:39:20,879 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/10c789d1aebb4595992fcbd7d141f22c to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/10c789d1aebb4595992fcbd7d141f22c 2024-11-25T03:39:20,880 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/24f5bae460c2407a8ba9d16d45ea22c4 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/24f5bae460c2407a8ba9d16d45ea22c4 2024-11-25T03:39:20,881 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/c6b6f01ded5f4c60aa7d4093e19c7d51 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/c6b6f01ded5f4c60aa7d4093e19c7d51 2024-11-25T03:39:20,882 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/2b66e9b80eb643cc9809dfca1de47920 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/2b66e9b80eb643cc9809dfca1de47920 2024-11-25T03:39:20,882 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/d20c677351d84b00888e0da1555e7a40 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/d20c677351d84b00888e0da1555e7a40 2024-11-25T03:39:20,883 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/7d7933614a8c47449bb3a2fe135c3738 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/7d7933614a8c47449bb3a2fe135c3738 2024-11-25T03:39:20,884 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/84624cbac1ed4ebfa508bf04846a1454 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/84624cbac1ed4ebfa508bf04846a1454 2024-11-25T03:39:20,885 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/9e6dfc9223da4882989e4786eb5ae2ee to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/9e6dfc9223da4882989e4786eb5ae2ee 2024-11-25T03:39:20,886 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/9ffe5326d6ce4c4eb668787eccd572a4 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/9ffe5326d6ce4c4eb668787eccd572a4 2024-11-25T03:39:20,886 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/fdcb53913c32436d9c3c2bfbc407bb7d to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/fdcb53913c32436d9c3c2bfbc407bb7d 2024-11-25T03:39:20,887 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/674a6f804c9f4f2a8051bb20e920376f to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/674a6f804c9f4f2a8051bb20e920376f 2024-11-25T03:39:20,888 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/74d25f312513457a9080b395614dffe7 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/74d25f312513457a9080b395614dffe7 2024-11-25T03:39:20,889 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/8a5a815fbd5a4213a674e7667e51053e to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/8a5a815fbd5a4213a674e7667e51053e 2024-11-25T03:39:20,890 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/18837405895c492d867015148033df07 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/18837405895c492d867015148033df07 2024-11-25T03:39:20,891 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/308033b80bb54bfabcf73933613953db to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/308033b80bb54bfabcf73933613953db 2024-11-25T03:39:20,891 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/8ad38a445e0e4bedaabdc006c66a0b3c to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/8ad38a445e0e4bedaabdc006c66a0b3c 2024-11-25T03:39:20,892 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/109407ffb22e4b08a613169948ba6549 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/109407ffb22e4b08a613169948ba6549 2024-11-25T03:39:20,893 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/e33981d1476240e98e719cd3588bfa6f to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/e33981d1476240e98e719cd3588bfa6f 2024-11-25T03:39:20,894 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/6481cf9ef563460fa1415098134f4cae to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/6481cf9ef563460fa1415098134f4cae 2024-11-25T03:39:20,895 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/2be075905ae044258b5b4d4c7b6a3a00 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/2be075905ae044258b5b4d4c7b6a3a00 2024-11-25T03:39:20,895 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/a0232b9f638f4f80b6c43f96a16be390 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/a0232b9f638f4f80b6c43f96a16be390 2024-11-25T03:39:20,896 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/bc964d8e432143af89f5b97489d20b94 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/bc964d8e432143af89f5b97489d20b94 2024-11-25T03:39:20,897 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/8eb22a1cfb004bd98a3c38fe23ea6791 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/8eb22a1cfb004bd98a3c38fe23ea6791 2024-11-25T03:39:20,898 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/9d6e0b52dfca43c9840b00ac77cd1183 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/9d6e0b52dfca43c9840b00ac77cd1183 2024-11-25T03:39:20,899 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/0a2b784783a4427facabd199889863e3 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/0a2b784783a4427facabd199889863e3 2024-11-25T03:39:20,900 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/d3f06f8b5d3840a99eeacde3f24d8655 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/d3f06f8b5d3840a99eeacde3f24d8655 2024-11-25T03:39:20,901 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/b5e64a2f085b4165878e5ab5ab7d57b4, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/911a6515ba45499fbbfb65cdb3d4410b, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/4bcbad7586df42b0acd295b972c2a872, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/6d0cd51effd84a1ea4ac8776df27d822, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/58d598f8a233470685fb1d3298123dd3, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/53e9ea64c32f444fa2dd5b6391badaf6, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/27e0f8fc0036462ebe4e3abd72b4be56, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/76ee702f29b24a08933d75bc990d85c3, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/462b6f87ba7446488db30b7d8f13a38d, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/466dded260544d99a29d6d6bca3a53f8, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/c28555b852e84629adaee83238c0ecc9, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/2ef3f68fc2f64c908f62c43a8ec038d9, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/a733e9b77e4c43a7a3aa97050c6a60c4, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/52bf1c0783b44f3794ed8294166b9c17, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/2f1dd915ee8948218fa833f21f299ee5, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/3f91a930ed884c919c81095ab3fa749c, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/c013869156624e529db8f2f96e3329eb, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/562a67fd9ae64f8a866039d6f86e41ab, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/1cb51df33573418e80c7b473896db18d, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/766fedfc2bfe47eb97e732797aa655e6, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/a2ff70455eb74ad2a7da681ecc1c6056, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/ad87fea86a7d4861b999d985afe6d266, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/641148dc4bf74a11b75f54998fee8e5e, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/fa2b57e788c14e92bf4f0095b022405e, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/e459daf8ce8048e2abf9482c88db8f47, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/f8af0af7e9394887a698b795efec92b7, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/7c793734468d4063bd3be7568f72fb9a] to archive 2024-11-25T03:39:20,901 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T03:39:20,902 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/b5e64a2f085b4165878e5ab5ab7d57b4 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/b5e64a2f085b4165878e5ab5ab7d57b4 2024-11-25T03:39:20,903 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/911a6515ba45499fbbfb65cdb3d4410b to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/911a6515ba45499fbbfb65cdb3d4410b 2024-11-25T03:39:20,904 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/4bcbad7586df42b0acd295b972c2a872 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/4bcbad7586df42b0acd295b972c2a872 2024-11-25T03:39:20,905 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/6d0cd51effd84a1ea4ac8776df27d822 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/6d0cd51effd84a1ea4ac8776df27d822 2024-11-25T03:39:20,905 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/58d598f8a233470685fb1d3298123dd3 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/58d598f8a233470685fb1d3298123dd3 2024-11-25T03:39:20,906 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/53e9ea64c32f444fa2dd5b6391badaf6 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/53e9ea64c32f444fa2dd5b6391badaf6 2024-11-25T03:39:20,907 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/27e0f8fc0036462ebe4e3abd72b4be56 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/27e0f8fc0036462ebe4e3abd72b4be56 2024-11-25T03:39:20,908 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/76ee702f29b24a08933d75bc990d85c3 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/76ee702f29b24a08933d75bc990d85c3 2024-11-25T03:39:20,909 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/462b6f87ba7446488db30b7d8f13a38d to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/462b6f87ba7446488db30b7d8f13a38d 2024-11-25T03:39:20,910 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/466dded260544d99a29d6d6bca3a53f8 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/466dded260544d99a29d6d6bca3a53f8 2024-11-25T03:39:20,910 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/c28555b852e84629adaee83238c0ecc9 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/c28555b852e84629adaee83238c0ecc9 2024-11-25T03:39:20,911 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/2ef3f68fc2f64c908f62c43a8ec038d9 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/2ef3f68fc2f64c908f62c43a8ec038d9 2024-11-25T03:39:20,912 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/a733e9b77e4c43a7a3aa97050c6a60c4 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/a733e9b77e4c43a7a3aa97050c6a60c4 2024-11-25T03:39:20,913 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/52bf1c0783b44f3794ed8294166b9c17 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/52bf1c0783b44f3794ed8294166b9c17 2024-11-25T03:39:20,914 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/2f1dd915ee8948218fa833f21f299ee5 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/2f1dd915ee8948218fa833f21f299ee5 2024-11-25T03:39:20,914 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/3f91a930ed884c919c81095ab3fa749c to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/3f91a930ed884c919c81095ab3fa749c 2024-11-25T03:39:20,915 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/c013869156624e529db8f2f96e3329eb to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/c013869156624e529db8f2f96e3329eb 2024-11-25T03:39:20,916 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/562a67fd9ae64f8a866039d6f86e41ab to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/562a67fd9ae64f8a866039d6f86e41ab 2024-11-25T03:39:20,917 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/1cb51df33573418e80c7b473896db18d to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/1cb51df33573418e80c7b473896db18d 2024-11-25T03:39:20,917 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/766fedfc2bfe47eb97e732797aa655e6 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/766fedfc2bfe47eb97e732797aa655e6 2024-11-25T03:39:20,918 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/a2ff70455eb74ad2a7da681ecc1c6056 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/a2ff70455eb74ad2a7da681ecc1c6056 2024-11-25T03:39:20,919 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/ad87fea86a7d4861b999d985afe6d266 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/ad87fea86a7d4861b999d985afe6d266 2024-11-25T03:39:20,920 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/641148dc4bf74a11b75f54998fee8e5e to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/641148dc4bf74a11b75f54998fee8e5e 2024-11-25T03:39:20,921 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/fa2b57e788c14e92bf4f0095b022405e to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/fa2b57e788c14e92bf4f0095b022405e 2024-11-25T03:39:20,921 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/e459daf8ce8048e2abf9482c88db8f47 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/e459daf8ce8048e2abf9482c88db8f47 2024-11-25T03:39:20,922 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/f8af0af7e9394887a698b795efec92b7 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/f8af0af7e9394887a698b795efec92b7 2024-11-25T03:39:20,923 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/7c793734468d4063bd3be7568f72fb9a to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/7c793734468d4063bd3be7568f72fb9a 2024-11-25T03:39:20,924 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/61b01d34d42041acae85a514a22605e7, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/10d63eac96ec4c73b2f3b9c11f8fa152, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/87965423dafa40b498cb23d0b2fa084c, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/80951d4328fa42359641f637bebe2b76, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/b7611f4544e74ed995edfda4943c4f36, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/0b92f8b53b39470dab67621c7bb0868e, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/011296115c204a66a8c575136e984999, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/9e776cbca2fb4120b18b2fd46ea0caba, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/120f35ef20bf44c5a7d614d4f87353ca, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/26958da9aa144e3eb7b722dd0d7847d3, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/2c2fa26a81fe4a28b96846948dc3a96a, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/a00c8cf665a547c8a8a1bcf067988582, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/d14f1d4953264608a69487c574258884, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/86250147737249259873a6303ab01547, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/19ee9d62a219490ea319e6d843600cdd, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/be7b105c673a491ebdf838f86d655d5b, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/2a806caf9a22449babc793ca31651a2e, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/c3b5ea28712f4d87be4fe397e072d935, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/2b6b429c48f24a1b9cbb989282a28cae, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/244ba37e6e35482894ed28983f493aaf, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/9c6b1f9fad1549ba92f283ca2c71542d, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/58ea85975dad48308bc4a8a289055ab3, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/3d7bfbeca6b3416d83ea189100233638, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/18fe05b806984002a2bae50a1f1bf049, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/e2bf703df961419f9db50244019dab35, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/58b3ae52cbf94bd3b7b5d91fd048397d, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/a32c97aadc954311a675467662e5d7cd] to archive 2024-11-25T03:39:20,925 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T03:39:20,926 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/61b01d34d42041acae85a514a22605e7 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/61b01d34d42041acae85a514a22605e7 2024-11-25T03:39:20,927 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/10d63eac96ec4c73b2f3b9c11f8fa152 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/10d63eac96ec4c73b2f3b9c11f8fa152 2024-11-25T03:39:20,927 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/87965423dafa40b498cb23d0b2fa084c to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/87965423dafa40b498cb23d0b2fa084c 2024-11-25T03:39:20,928 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/80951d4328fa42359641f637bebe2b76 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/80951d4328fa42359641f637bebe2b76 2024-11-25T03:39:20,929 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/b7611f4544e74ed995edfda4943c4f36 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/b7611f4544e74ed995edfda4943c4f36 2024-11-25T03:39:20,930 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/0b92f8b53b39470dab67621c7bb0868e to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/0b92f8b53b39470dab67621c7bb0868e 2024-11-25T03:39:20,930 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/011296115c204a66a8c575136e984999 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/011296115c204a66a8c575136e984999 2024-11-25T03:39:20,931 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/9e776cbca2fb4120b18b2fd46ea0caba to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/9e776cbca2fb4120b18b2fd46ea0caba 2024-11-25T03:39:20,932 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/120f35ef20bf44c5a7d614d4f87353ca to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/120f35ef20bf44c5a7d614d4f87353ca 2024-11-25T03:39:20,932 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/26958da9aa144e3eb7b722dd0d7847d3 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/26958da9aa144e3eb7b722dd0d7847d3 2024-11-25T03:39:20,933 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/2c2fa26a81fe4a28b96846948dc3a96a to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/2c2fa26a81fe4a28b96846948dc3a96a 2024-11-25T03:39:20,934 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/a00c8cf665a547c8a8a1bcf067988582 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/a00c8cf665a547c8a8a1bcf067988582 2024-11-25T03:39:20,935 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/d14f1d4953264608a69487c574258884 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/d14f1d4953264608a69487c574258884 2024-11-25T03:39:20,935 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/86250147737249259873a6303ab01547 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/86250147737249259873a6303ab01547 2024-11-25T03:39:20,936 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/19ee9d62a219490ea319e6d843600cdd to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/19ee9d62a219490ea319e6d843600cdd 2024-11-25T03:39:20,937 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/be7b105c673a491ebdf838f86d655d5b to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/be7b105c673a491ebdf838f86d655d5b 2024-11-25T03:39:20,937 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/2a806caf9a22449babc793ca31651a2e to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/2a806caf9a22449babc793ca31651a2e 2024-11-25T03:39:20,938 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/c3b5ea28712f4d87be4fe397e072d935 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/c3b5ea28712f4d87be4fe397e072d935 2024-11-25T03:39:20,939 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/2b6b429c48f24a1b9cbb989282a28cae to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/2b6b429c48f24a1b9cbb989282a28cae 2024-11-25T03:39:20,940 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/244ba37e6e35482894ed28983f493aaf to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/244ba37e6e35482894ed28983f493aaf 2024-11-25T03:39:20,941 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/9c6b1f9fad1549ba92f283ca2c71542d to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/9c6b1f9fad1549ba92f283ca2c71542d 2024-11-25T03:39:20,942 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/58ea85975dad48308bc4a8a289055ab3 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/58ea85975dad48308bc4a8a289055ab3 2024-11-25T03:39:20,942 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/3d7bfbeca6b3416d83ea189100233638 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/3d7bfbeca6b3416d83ea189100233638 2024-11-25T03:39:20,943 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/18fe05b806984002a2bae50a1f1bf049 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/18fe05b806984002a2bae50a1f1bf049 2024-11-25T03:39:20,944 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/e2bf703df961419f9db50244019dab35 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/e2bf703df961419f9db50244019dab35 2024-11-25T03:39:20,945 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/58b3ae52cbf94bd3b7b5d91fd048397d to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/58b3ae52cbf94bd3b7b5d91fd048397d 2024-11-25T03:39:20,945 DEBUG [StoreCloser-TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/a32c97aadc954311a675467662e5d7cd to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/a32c97aadc954311a675467662e5d7cd 2024-11-25T03:39:20,949 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/recovered.edits/389.seqid, newMaxSeqId=389, maxSeqId=1 2024-11-25T03:39:20,950 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. 2024-11-25T03:39:20,950 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1676): Region close journal for 502b5a21115d6420c81225a0df454848: Waiting for close lock at 1732505959639Running coprocessor pre-close hooks at 1732505959639Disabling compacts and flushes for region at 1732505959639Disabling writes for close at 1732505959639Obtaining lock to block concurrent updates at 1732505959640 (+1 ms)Preparing flush snapshotting stores in 502b5a21115d6420c81225a0df454848 at 1732505959640Finished memstore snapshotting TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1732505959640Flushing stores of TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848. at 1732505959640Flushing 502b5a21115d6420c81225a0df454848/A: creating writer at 1732505959640Flushing 502b5a21115d6420c81225a0df454848/A: appending metadata at 1732505959642 (+2 ms)Flushing 502b5a21115d6420c81225a0df454848/A: closing flushed file at 1732505959642Flushing 502b5a21115d6420c81225a0df454848/B: creating writer at 1732505960050 (+408 ms)Flushing 502b5a21115d6420c81225a0df454848/B: appending metadata at 1732505960052 (+2 ms)Flushing 502b5a21115d6420c81225a0df454848/B: closing flushed file at 1732505960052Flushing 502b5a21115d6420c81225a0df454848/C: creating writer at 1732505960458 (+406 ms)Flushing 502b5a21115d6420c81225a0df454848/C: appending metadata at 1732505960460 (+2 ms)Flushing 502b5a21115d6420c81225a0df454848/C: closing flushed file at 1732505960460Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a6e1b32: reopening flushed file at 1732505960867 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c98a6c4: reopening flushed file at 1732505960870 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38ad33e3: reopening flushed file at 1732505960873 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 502b5a21115d6420c81225a0df454848 in 1237ms, sequenceid=386, compaction requested=false at 1732505960876 (+3 ms)Writing region close event to WAL at 1732505960946 (+70 ms)Running coprocessor post-close hooks at 1732505960949 (+3 ms)Closed at 1732505960950 (+1 ms) 2024-11-25T03:39:20,951 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(157): Closed 502b5a21115d6420c81225a0df454848 2024-11-25T03:39:20,952 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=502b5a21115d6420c81225a0df454848, regionState=CLOSED 2024-11-25T03:39:20,953 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure 502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 because future has completed 2024-11-25T03:39:20,955 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-25T03:39:20,955 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseRegionProcedure 502b5a21115d6420c81225a0df454848, server=38a5d2077268,34881,1732505806588 in 1.4660 sec 2024-11-25T03:39:20,956 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-11-25T03:39:20,956 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=502b5a21115d6420c81225a0df454848, UNASSIGN in 1.4700 sec 2024-11-25T03:39:20,957 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-25T03:39:20,957 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4720 sec 2024-11-25T03:39:20,958 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732505960958"}]},"ts":"1732505960958"} 2024-11-25T03:39:20,959 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-25T03:39:20,959 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-25T03:39:20,961 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4800 sec 2024-11-25T03:39:21,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-25T03:39:21,628 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:39:21,628 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-25T03:39:21,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:39:21,630 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=139, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:39:21,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-25T03:39:21,630 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=139, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:39:21,632 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848 2024-11-25T03:39:21,634 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A, FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B, FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C, FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/recovered.edits] 2024-11-25T03:39:21,636 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/94fd374a1e9d454bb39d11f2847ff19d to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/94fd374a1e9d454bb39d11f2847ff19d 2024-11-25T03:39:21,636 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/bd6e2141e8fe46b8b4c1b2ee214ef0d0 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/A/bd6e2141e8fe46b8b4c1b2ee214ef0d0 2024-11-25T03:39:21,638 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/8adfa28bc7fd42c9bf056a318e40a4b4 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/8adfa28bc7fd42c9bf056a318e40a4b4 2024-11-25T03:39:21,639 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/e4614e329eb84b36a12bdbd8afe76f17 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/B/e4614e329eb84b36a12bdbd8afe76f17 2024-11-25T03:39:21,641 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/352a2af853664b689edff6e3f547f381 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/352a2af853664b689edff6e3f547f381 2024-11-25T03:39:21,641 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/ea396025d4184fe888fca491aaffa996 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/C/ea396025d4184fe888fca491aaffa996 2024-11-25T03:39:21,643 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/recovered.edits/389.seqid to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848/recovered.edits/389.seqid 2024-11-25T03:39:21,644 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/502b5a21115d6420c81225a0df454848 2024-11-25T03:39:21,644 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-25T03:39:21,646 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=139, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:39:21,647 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-25T03:39:21,649 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-25T03:39:21,650 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=139, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:39:21,650 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-25T03:39:21,650 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732505961650"}]},"ts":"9223372036854775807"} 2024-11-25T03:39:21,652 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-25T03:39:21,652 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 502b5a21115d6420c81225a0df454848, NAME => 'TestAcidGuarantees,,1732505931156.502b5a21115d6420c81225a0df454848.', STARTKEY => '', ENDKEY => ''}] 2024-11-25T03:39:21,652 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-25T03:39:21,652 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732505961652"}]},"ts":"9223372036854775807"} 2024-11-25T03:39:21,653 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-25T03:39:21,654 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=139, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:39:21,654 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 26 msec 2024-11-25T03:39:21,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-25T03:39:21,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-25T03:39:21,740 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:39:21,747 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=238 (was 242), OpenFileDescriptor=449 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=911 (was 1321), ProcessCount=11 (was 11), AvailableMemoryMB=6639 (was 6679) 2024-11-25T03:39:21,755 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=238, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=911, ProcessCount=11, AvailableMemoryMB=6639 2024-11-25T03:39:21,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T03:39:21,756 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T03:39:21,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=140, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-25T03:39:21,758 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T03:39:21,758 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 140 2024-11-25T03:39:21,758 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:21,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-25T03:39:21,759 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T03:39:21,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742321_1497 (size=959) 2024-11-25T03:39:21,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-25T03:39:22,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-25T03:39:22,165 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3 2024-11-25T03:39:22,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742322_1498 (size=53) 2024-11-25T03:39:22,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-25T03:39:22,571 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:39:22,571 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 33de131a28520cc9ecc2bf159dfd6128, disabling compactions & flushes 2024-11-25T03:39:22,571 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:22,571 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:22,571 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. after waiting 0 ms 2024-11-25T03:39:22,571 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:22,571 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:22,571 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 33de131a28520cc9ecc2bf159dfd6128: Waiting for close lock at 1732505962571Disabling compacts and flushes for region at 1732505962571Disabling writes for close at 1732505962571Writing region close event to WAL at 1732505962571Closed at 1732505962571 2024-11-25T03:39:22,572 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T03:39:22,572 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732505962572"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732505962572"}]},"ts":"1732505962572"} 2024-11-25T03:39:22,573 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T03:39:22,574 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T03:39:22,574 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732505962574"}]},"ts":"1732505962574"} 2024-11-25T03:39:22,575 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-25T03:39:22,575 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=33de131a28520cc9ecc2bf159dfd6128, ASSIGN}] 2024-11-25T03:39:22,576 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=33de131a28520cc9ecc2bf159dfd6128, ASSIGN 2024-11-25T03:39:22,576 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=33de131a28520cc9ecc2bf159dfd6128, ASSIGN; state=OFFLINE, location=38a5d2077268,34881,1732505806588; forceNewPlan=false, retain=false 2024-11-25T03:39:22,727 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=33de131a28520cc9ecc2bf159dfd6128, regionState=OPENING, regionLocation=38a5d2077268,34881,1732505806588 2024-11-25T03:39:22,728 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=33de131a28520cc9ecc2bf159dfd6128, ASSIGN because future has completed 2024-11-25T03:39:22,728 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure 33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588}] 2024-11-25T03:39:22,882 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:22,883 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7752): Opening region: {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:39:22,883 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:22,883 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:39:22,883 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7794): checking encryption for 33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:22,883 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7797): checking classloading for 33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:22,884 INFO [StoreOpener-33de131a28520cc9ecc2bf159dfd6128-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:22,885 INFO [StoreOpener-33de131a28520cc9ecc2bf159dfd6128-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 33de131a28520cc9ecc2bf159dfd6128 columnFamilyName A 2024-11-25T03:39:22,885 DEBUG [StoreOpener-33de131a28520cc9ecc2bf159dfd6128-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:22,886 INFO [StoreOpener-33de131a28520cc9ecc2bf159dfd6128-1 {}] regionserver.HStore(327): Store=33de131a28520cc9ecc2bf159dfd6128/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:39:22,886 INFO [StoreOpener-33de131a28520cc9ecc2bf159dfd6128-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:22,887 INFO [StoreOpener-33de131a28520cc9ecc2bf159dfd6128-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 33de131a28520cc9ecc2bf159dfd6128 columnFamilyName B 2024-11-25T03:39:22,887 DEBUG [StoreOpener-33de131a28520cc9ecc2bf159dfd6128-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:22,887 INFO [StoreOpener-33de131a28520cc9ecc2bf159dfd6128-1 {}] regionserver.HStore(327): Store=33de131a28520cc9ecc2bf159dfd6128/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:39:22,887 INFO [StoreOpener-33de131a28520cc9ecc2bf159dfd6128-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:22,888 INFO [StoreOpener-33de131a28520cc9ecc2bf159dfd6128-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 33de131a28520cc9ecc2bf159dfd6128 columnFamilyName C 2024-11-25T03:39:22,888 DEBUG [StoreOpener-33de131a28520cc9ecc2bf159dfd6128-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:22,888 INFO [StoreOpener-33de131a28520cc9ecc2bf159dfd6128-1 {}] regionserver.HStore(327): Store=33de131a28520cc9ecc2bf159dfd6128/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:39:22,888 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1038): replaying wal for 33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:22,889 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:22,889 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:22,890 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1048): stopping wal replay for 33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:22,890 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1060): Cleaning up temporary data for 33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:22,890 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T03:39:22,891 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1093): writing seq id for 33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:22,893 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:39:22,893 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1114): Opened 33de131a28520cc9ecc2bf159dfd6128; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59634664, jitterRate=-0.11137425899505615}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T03:39:22,893 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:22,894 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1006): Region open journal for 33de131a28520cc9ecc2bf159dfd6128: Running coprocessor pre-open hook at 1732505962883Writing region info on filesystem at 1732505962883Initializing all the Stores at 1732505962884 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732505962884Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732505962884Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732505962884Cleaning up temporary data from old regions at 1732505962890 (+6 ms)Running coprocessor post-open hooks at 1732505962893 (+3 ms)Region opened successfully at 1732505962894 (+1 ms) 2024-11-25T03:39:22,894 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., pid=142, masterSystemTime=1732505962880 2024-11-25T03:39:22,896 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:22,896 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:22,896 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=33de131a28520cc9ecc2bf159dfd6128, regionState=OPEN, openSeqNum=2, regionLocation=38a5d2077268,34881,1732505806588 2024-11-25T03:39:22,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-25T03:39:22,897 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure 33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 because future has completed 2024-11-25T03:39:22,899 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-25T03:39:22,899 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; OpenRegionProcedure 33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 in 170 msec 2024-11-25T03:39:22,900 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-25T03:39:22,900 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=33de131a28520cc9ecc2bf159dfd6128, ASSIGN in 324 msec 2024-11-25T03:39:22,901 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T03:39:22,901 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732505962901"}]},"ts":"1732505962901"} 2024-11-25T03:39:22,902 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-25T03:39:22,902 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T03:39:22,903 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1460 sec 2024-11-25T03:39:23,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-25T03:39:23,907 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:39:23,908 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e76e6cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:39:23,908 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:39:23,909 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:39:23,909 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:39:23,910 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:39:23,910 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:39:23,910 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e1b93a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:39:23,910 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:39:23,910 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:39:23,910 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:23,911 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43540, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:39:23,911 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ac07d46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:39:23,912 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:39:23,912 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:39:23,913 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:39:23,913 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39846, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:39:23,914 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T03:39:23,915 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 38a5d2077268,42243,1732505805780 2024-11-25T03:39:23,915 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@57c88759 2024-11-25T03:39:23,915 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T03:39:23,915 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43548, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T03:39:23,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T03:39:23,917 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T03:39:23,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-25T03:39:23,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742323_1499 (size=995) 2024-11-25T03:39:24,327 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-25T03:39:24,328 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-25T03:39:24,330 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T03:39:24,331 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=33de131a28520cc9ecc2bf159dfd6128, REOPEN/MOVE}] 2024-11-25T03:39:24,332 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=33de131a28520cc9ecc2bf159dfd6128, REOPEN/MOVE 2024-11-25T03:39:24,332 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=33de131a28520cc9ecc2bf159dfd6128, regionState=CLOSING, regionLocation=38a5d2077268,34881,1732505806588 2024-11-25T03:39:24,333 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=33de131a28520cc9ecc2bf159dfd6128, REOPEN/MOVE because future has completed 2024-11-25T03:39:24,334 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T03:39:24,334 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure 33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588}] 2024-11-25T03:39:24,485 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(122): Close 33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:24,486 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T03:39:24,486 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1722): Closing 33de131a28520cc9ecc2bf159dfd6128, disabling compactions & flushes 2024-11-25T03:39:24,486 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:24,486 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:24,486 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. after waiting 0 ms 2024-11-25T03:39:24,486 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:24,489 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-25T03:39:24,489 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:24,489 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1676): Region close journal for 33de131a28520cc9ecc2bf159dfd6128: Waiting for close lock at 1732505964486Running coprocessor pre-close hooks at 1732505964486Disabling compacts and flushes for region at 1732505964486Disabling writes for close at 1732505964486Writing region close event to WAL at 1732505964486Running coprocessor post-close hooks at 1732505964489 (+3 ms)Closed at 1732505964489 2024-11-25T03:39:24,489 WARN [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionServer(3299): Not adding moved region record: 33de131a28520cc9ecc2bf159dfd6128 to self. 2024-11-25T03:39:24,491 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(157): Closed 33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:24,491 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=33de131a28520cc9ecc2bf159dfd6128, regionState=CLOSED 2024-11-25T03:39:24,492 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure 33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 because future has completed 2024-11-25T03:39:24,494 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-25T03:39:24,494 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; CloseRegionProcedure 33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 in 159 msec 2024-11-25T03:39:24,495 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=33de131a28520cc9ecc2bf159dfd6128, REOPEN/MOVE; state=CLOSED, location=38a5d2077268,34881,1732505806588; forceNewPlan=false, retain=true 2024-11-25T03:39:24,645 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=33de131a28520cc9ecc2bf159dfd6128, regionState=OPENING, regionLocation=38a5d2077268,34881,1732505806588 2024-11-25T03:39:24,647 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=33de131a28520cc9ecc2bf159dfd6128, REOPEN/MOVE because future has completed 2024-11-25T03:39:24,647 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588}] 2024-11-25T03:39:24,801 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:24,801 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7752): Opening region: {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:39:24,802 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:24,802 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:39:24,802 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7794): checking encryption for 33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:24,802 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7797): checking classloading for 33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:24,803 INFO [StoreOpener-33de131a28520cc9ecc2bf159dfd6128-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:24,803 INFO [StoreOpener-33de131a28520cc9ecc2bf159dfd6128-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 33de131a28520cc9ecc2bf159dfd6128 columnFamilyName A 2024-11-25T03:39:24,804 DEBUG [StoreOpener-33de131a28520cc9ecc2bf159dfd6128-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:24,805 INFO [StoreOpener-33de131a28520cc9ecc2bf159dfd6128-1 {}] regionserver.HStore(327): Store=33de131a28520cc9ecc2bf159dfd6128/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:39:24,805 INFO [StoreOpener-33de131a28520cc9ecc2bf159dfd6128-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:24,805 INFO [StoreOpener-33de131a28520cc9ecc2bf159dfd6128-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 33de131a28520cc9ecc2bf159dfd6128 columnFamilyName B 2024-11-25T03:39:24,805 DEBUG [StoreOpener-33de131a28520cc9ecc2bf159dfd6128-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:24,806 INFO [StoreOpener-33de131a28520cc9ecc2bf159dfd6128-1 {}] regionserver.HStore(327): Store=33de131a28520cc9ecc2bf159dfd6128/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:39:24,806 INFO [StoreOpener-33de131a28520cc9ecc2bf159dfd6128-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:24,806 INFO [StoreOpener-33de131a28520cc9ecc2bf159dfd6128-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 33de131a28520cc9ecc2bf159dfd6128 columnFamilyName C 2024-11-25T03:39:24,806 DEBUG [StoreOpener-33de131a28520cc9ecc2bf159dfd6128-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:24,807 INFO [StoreOpener-33de131a28520cc9ecc2bf159dfd6128-1 {}] regionserver.HStore(327): Store=33de131a28520cc9ecc2bf159dfd6128/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:39:24,807 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1038): replaying wal for 33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:24,807 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:24,808 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:24,808 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1048): stopping wal replay for 33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:24,808 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1060): Cleaning up temporary data for 33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:24,809 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T03:39:24,809 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1093): writing seq id for 33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:24,810 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1114): Opened 33de131a28520cc9ecc2bf159dfd6128; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71711932, jitterRate=0.06859105825424194}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T03:39:24,810 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:24,810 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1006): Region open journal for 33de131a28520cc9ecc2bf159dfd6128: Running coprocessor pre-open hook at 1732505964802Writing region info on filesystem at 1732505964802Initializing all the Stores at 1732505964802Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732505964802Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732505964803 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732505964803Cleaning up temporary data from old regions at 1732505964808 (+5 ms)Running coprocessor post-open hooks at 1732505964810 (+2 ms)Region opened successfully at 1732505964810 2024-11-25T03:39:24,811 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., pid=147, masterSystemTime=1732505964799 2024-11-25T03:39:24,812 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:24,813 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:24,813 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=33de131a28520cc9ecc2bf159dfd6128, regionState=OPEN, openSeqNum=5, regionLocation=38a5d2077268,34881,1732505806588 2024-11-25T03:39:24,814 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 because future has completed 2024-11-25T03:39:24,816 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=145 2024-11-25T03:39:24,816 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=145, state=SUCCESS, hasLock=false; OpenRegionProcedure 33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 in 168 msec 2024-11-25T03:39:24,818 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-25T03:39:24,818 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=33de131a28520cc9ecc2bf159dfd6128, REOPEN/MOVE in 485 msec 2024-11-25T03:39:24,820 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-25T03:39:24,820 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 489 msec 2024-11-25T03:39:24,821 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 903 msec 2024-11-25T03:39:24,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-25T03:39:24,822 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:39:24,822 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e2aa61c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:39:24,822 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:39:24,823 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:39:24,823 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:39:24,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:39:24,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:39:24,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37d0fad5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:39:24,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:39:24,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:39:24,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:24,825 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43560, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:39:24,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fb3f2d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:39:24,826 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ca6c80d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:39:24,826 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:39:24,826 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:39:24,826 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:39:24,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:39:24,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:39:24,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@781e74c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:39:24,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:39:24,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:39:24,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:24,827 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43578, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:39:24,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24a195fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:39:24,828 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14afc90b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:39:24,828 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:39:24,828 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:39:24,829 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:39:24,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:39:24,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:39:24,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51f6d656, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:39:24,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:39:24,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:39:24,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:24,830 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43606, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:39:24,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36b82b95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:39:24,831 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47173f60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:39:24,831 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:39:24,831 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:39:24,831 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:39:24,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:39:24,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:39:24,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c7280ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:39:24,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:39:24,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:39:24,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:24,833 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43626, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:39:24,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a4871, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:39:24,833 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13700528, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:39:24,833 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:39:24,834 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:39:24,834 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:39:24,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:39:24,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:39:24,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2deb1ca9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:39:24,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:39:24,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:39:24,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:24,835 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43644, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:39:24,835 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cf36d7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:39:24,836 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3666b173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:39:24,836 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:39:24,836 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:39:24,837 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:39:24,837 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:39:24,837 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:39:24,837 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ed8de6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:39:24,837 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:39:24,837 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:39:24,837 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:24,838 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43658, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:39:24,838 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dfa741c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:39:24,839 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77ff027c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:39:24,839 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:39:24,839 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:39:24,839 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:39:24,839 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:39:24,839 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:39:24,839 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f470959, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:39:24,839 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:39:24,840 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:39:24,840 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:24,840 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43680, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:39:24,841 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a0167c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:39:24,841 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2637386d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:39:24,841 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:39:24,841 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:39:24,842 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:39:24,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:39:24,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:39:24,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72f72ee9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:39:24,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:39:24,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:39:24,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:24,843 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43700, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:39:24,843 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c257f44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:39:24,844 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2455dde6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:39:24,844 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:39:24,844 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:39:24,845 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:39:24,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:39:24,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:39:24,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a23844a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:39:24,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:39:24,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:39:24,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:24,846 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43722, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:39:24,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24705f9f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:39:24,847 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b93642c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:39:24,847 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,42243,-1 for getting cluster id 2024-11-25T03:39:24,847 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:39:24,847 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd693e68b-1234-49de-b9a6-51eec9db27cc' 2024-11-25T03:39:24,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:39:24,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d693e68b-1234-49de-b9a6-51eec9db27cc" 2024-11-25T03:39:24,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75b78816, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:39:24,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,42243,-1] 2024-11-25T03:39:24,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:39:24,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:24,848 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43752, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:39:24,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d58236f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:39:24,849 DEBUG [Thread-2263 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:39:24,849 DEBUG [Thread-2261 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:39:24,849 DEBUG [Thread-2254 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:39:24,850 DEBUG [Thread-2250 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:39:24,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:39:24,850 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:39:24,850 DEBUG [Thread-2248 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:39:24,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:39:24,850 DEBUG [Thread-2252 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:39:24,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:39:24,850 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:39:24,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:39:24,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:39:24,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:39:24,851 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:39:24,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:39:24,851 DEBUG [Thread-2256 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:39:24,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:39:24,851 DEBUG [Thread-2265 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:39:24,851 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39860, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:39:24,851 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39858, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:39:24,852 DEBUG [Thread-2259 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:39:24,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:39:24,852 DEBUG [Thread-2267 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:39:24,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:39:24,852 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39884, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:39:24,852 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39854, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:39:24,852 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39882, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:39:24,852 DEBUG [Thread-2252 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:39:24,852 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:39:24,853 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:39:24,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-25T03:39:24,853 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:39:24,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,34881,1732505806588, seqNum=-1] 2024-11-25T03:39:24,853 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:39:24,853 DEBUG [Thread-2252 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:39:24,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:39:24,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-25T03:39:24,854 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39900, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:39:24,854 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39892, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:39:24,854 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39904, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:39:24,854 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39920, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:39:24,854 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:39:24,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5] 2024-11-25T03:39:24,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5] 2024-11-25T03:39:24,855 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39936, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:39:24,855 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:39:24,855 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:39:24,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5] 2024-11-25T03:39:24,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5] 2024-11-25T03:39:24,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5] 2024-11-25T03:39:24,858 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5] 2024-11-25T03:39:24,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5] 2024-11-25T03:39:24,858 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5] 2024-11-25T03:39:24,859 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5] 2024-11-25T03:39:24,860 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5] 2024-11-25T03:39:24,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:24,864 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:39:24,864 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 33de131a28520cc9ecc2bf159dfd6128 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-25T03:39:24,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:24,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39858 deadline: 1732506024883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:24,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:24,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:24,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:24,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1732506024884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:24,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:24,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:24,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:24,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:24,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:24,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1732506024887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:24,891 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:24,891 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:24,891 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:24,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:24,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39884 deadline: 1732506024888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:24,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:24,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:24,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:24,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39900 deadline: 1732506024889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:24,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:24,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:24,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:24,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:24,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112585c554dce98544c380913606389c018b_33de131a28520cc9ecc2bf159dfd6128 is 50, key is test_row_0/A:col10/1732505964862/Put/seqid=0 2024-11-25T03:39:24,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742324_1500 (size=14594) 2024-11-25T03:39:24,922 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:24,926 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112585c554dce98544c380913606389c018b_33de131a28520cc9ecc2bf159dfd6128 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112585c554dce98544c380913606389c018b_33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:24,927 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/6ad3d64a9c404233b1730e18f65d2024, store: [table=TestAcidGuarantees family=A region=33de131a28520cc9ecc2bf159dfd6128] 2024-11-25T03:39:24,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/6ad3d64a9c404233b1730e18f65d2024 is 175, key is test_row_0/A:col10/1732505964862/Put/seqid=0 2024-11-25T03:39:24,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742325_1501 (size=39549) 2024-11-25T03:39:24,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-25T03:39:24,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:24,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39858 deadline: 1732506024997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:25,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:25,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:25,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:25,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1732506024997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:25,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:25,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:25,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:25,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:25,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:25,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1732506024997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:25,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:25,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:25,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:25,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:25,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39884 deadline: 1732506024997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:25,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:25,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:25,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:25,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:25,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39900 deadline: 1732506024997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:25,001 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:25,001 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:25,001 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:25,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-25T03:39:25,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:25,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. as already flushing 2024-11-25T03:39:25,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:25,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:25,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:25,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:25,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-25T03:39:25,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:25,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. as already flushing 2024-11-25T03:39:25,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:25,161 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:25,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:25,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:25,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-25T03:39:25,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:25,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39858 deadline: 1732506025207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:25,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:25,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:25,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1732506025207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:25,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:25,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:25,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:25,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:25,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:25,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:25,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1732506025207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:25,209 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:25,209 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:25,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:25,209 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:25,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39884 deadline: 1732506025207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:25,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:25,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:25,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:25,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:25,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39900 deadline: 1732506025208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:25,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:25,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:25,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:25,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-25T03:39:25,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:25,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. as already flushing 2024-11-25T03:39:25,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:25,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:25,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:25,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:25,340 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/6ad3d64a9c404233b1730e18f65d2024 2024-11-25T03:39:25,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/d154319122014d238fc3a0508af7c668 is 50, key is test_row_0/B:col10/1732505964862/Put/seqid=0 2024-11-25T03:39:25,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742326_1502 (size=12001) 2024-11-25T03:39:25,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-25T03:39:25,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:25,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. as already flushing 2024-11-25T03:39:25,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:25,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:25,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:25,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:25,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-25T03:39:25,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:25,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39858 deadline: 1732506025517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:25,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:25,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1732506025517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:25,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:25,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:25,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:25,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:25,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39884 deadline: 1732506025517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:25,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:25,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:25,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:25,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:25,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:25,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:25,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:25,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1732506025517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:25,521 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:25,521 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:25,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:25,521 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:25,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39900 deadline: 1732506025517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:25,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:25,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:25,522 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:25,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-25T03:39:25,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:25,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. as already flushing 2024-11-25T03:39:25,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:25,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:25,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:25,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:25,768 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/d154319122014d238fc3a0508af7c668 2024-11-25T03:39:25,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-25T03:39:25,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:25,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. as already flushing 2024-11-25T03:39:25,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:25,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:25,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:25,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:25,791 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/fe66b944adce47ee9dd2530b2fec1106 is 50, key is test_row_0/C:col10/1732505964862/Put/seqid=0 2024-11-25T03:39:25,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742327_1503 (size=12001) 2024-11-25T03:39:25,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-25T03:39:25,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:25,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. as already flushing 2024-11-25T03:39:25,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:25,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:25,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:25,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:25,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-25T03:39:26,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:26,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39858 deadline: 1732506026027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:26,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:26,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:26,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1732506026027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:26,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39884 deadline: 1732506026027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:26,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:26,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:26,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:26,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:26,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:26,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:26,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:26,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:26,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:26,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1732506026027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:26,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:26,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:26,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39900 deadline: 1732506026028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:26,031 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:26,031 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:26,031 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:26,031 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:26,031 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:26,031 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:26,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-25T03:39:26,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:26,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. as already flushing 2024-11-25T03:39:26,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:26,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:26,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:26,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:26,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/fe66b944adce47ee9dd2530b2fec1106 2024-11-25T03:39:26,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/6ad3d64a9c404233b1730e18f65d2024 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/6ad3d64a9c404233b1730e18f65d2024 2024-11-25T03:39:26,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/6ad3d64a9c404233b1730e18f65d2024, entries=200, sequenceid=16, filesize=38.6 K 2024-11-25T03:39:26,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/d154319122014d238fc3a0508af7c668 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/d154319122014d238fc3a0508af7c668 2024-11-25T03:39:26,211 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/d154319122014d238fc3a0508af7c668, entries=150, sequenceid=16, filesize=11.7 K 2024-11-25T03:39:26,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/fe66b944adce47ee9dd2530b2fec1106 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/fe66b944adce47ee9dd2530b2fec1106 2024-11-25T03:39:26,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/fe66b944adce47ee9dd2530b2fec1106, entries=150, sequenceid=16, filesize=11.7 K 2024-11-25T03:39:26,215 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 33de131a28520cc9ecc2bf159dfd6128 in 1351ms, sequenceid=16, compaction requested=false 2024-11-25T03:39:26,215 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-25T03:39:26,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 33de131a28520cc9ecc2bf159dfd6128: 2024-11-25T03:39:26,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-25T03:39:26,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:26,232 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 33de131a28520cc9ecc2bf159dfd6128 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-25T03:39:26,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125615ccdfc13ad42ea83536bb8f6a4d5e8_33de131a28520cc9ecc2bf159dfd6128 is 50, key is test_row_0/A:col10/1732505964887/Put/seqid=0 2024-11-25T03:39:26,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742328_1504 (size=12154) 2024-11-25T03:39:26,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:26,246 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125615ccdfc13ad42ea83536bb8f6a4d5e8_33de131a28520cc9ecc2bf159dfd6128 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125615ccdfc13ad42ea83536bb8f6a4d5e8_33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:26,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/7f1037a2539447409d0156d2b5b4d7a9, store: [table=TestAcidGuarantees family=A region=33de131a28520cc9ecc2bf159dfd6128] 2024-11-25T03:39:26,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/7f1037a2539447409d0156d2b5b4d7a9 is 175, key is test_row_0/A:col10/1732505964887/Put/seqid=0 2024-11-25T03:39:26,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742329_1505 (size=30955) 2024-11-25T03:39:26,252 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/7f1037a2539447409d0156d2b5b4d7a9 2024-11-25T03:39:26,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/b4c6ba40e98f4b6f94c8974a07cc1198 is 50, key is test_row_0/B:col10/1732505964887/Put/seqid=0 2024-11-25T03:39:26,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742330_1506 (size=12001) 2024-11-25T03:39:26,468 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T03:39:26,676 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/b4c6ba40e98f4b6f94c8974a07cc1198 2024-11-25T03:39:26,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/c6d87f70a45e42a48877552edc5687d3 is 50, key is test_row_0/C:col10/1732505964887/Put/seqid=0 2024-11-25T03:39:26,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742331_1507 (size=12001) 2024-11-25T03:39:26,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-25T03:39:27,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:27,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. as already flushing 2024-11-25T03:39:27,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:27,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1732506027047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:27,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:27,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39900 deadline: 1732506027048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:27,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:27,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1732506027048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:27,052 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,34881,1732505806588 for 'test_row_0' in 33de131a28520cc9ecc2bf159dfd6128 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:39:27,052 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,34881,1732505806588 for 'test_row_2' in 33de131a28520cc9ecc2bf159dfd6128 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:39:27,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:27,052 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:27,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:27,053 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:27,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:27,053 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:27,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:27,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:27,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:27,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39858 deadline: 1732506027050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:27,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:27,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:27,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:27,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39884 deadline: 1732506027051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:27,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:27,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:27,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:27,086 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/c6d87f70a45e42a48877552edc5687d3 2024-11-25T03:39:27,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/7f1037a2539447409d0156d2b5b4d7a9 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/7f1037a2539447409d0156d2b5b4d7a9 2024-11-25T03:39:27,093 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/7f1037a2539447409d0156d2b5b4d7a9, entries=150, sequenceid=41, filesize=30.2 K 2024-11-25T03:39:27,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/b4c6ba40e98f4b6f94c8974a07cc1198 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/b4c6ba40e98f4b6f94c8974a07cc1198 2024-11-25T03:39:27,098 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/b4c6ba40e98f4b6f94c8974a07cc1198, entries=150, sequenceid=41, filesize=11.7 K 2024-11-25T03:39:27,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/c6d87f70a45e42a48877552edc5687d3 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/c6d87f70a45e42a48877552edc5687d3 2024-11-25T03:39:27,102 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/c6d87f70a45e42a48877552edc5687d3, entries=150, sequenceid=41, filesize=11.7 K 2024-11-25T03:39:27,102 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 33de131a28520cc9ecc2bf159dfd6128 in 870ms, sequenceid=41, compaction requested=false 2024-11-25T03:39:27,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 33de131a28520cc9ecc2bf159dfd6128: 2024-11-25T03:39:27,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:27,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-25T03:39:27,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-25T03:39:27,106 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-25T03:39:27,106 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2490 sec 2024-11-25T03:39:27,107 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 2.2550 sec 2024-11-25T03:39:27,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:27,160 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:39:27,160 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 33de131a28520cc9ecc2bf159dfd6128 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-25T03:39:27,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125dc72c12b206644a6957a02dd15314092_33de131a28520cc9ecc2bf159dfd6128 is 50, key is test_row_0/A:col10/1732505967158/Put/seqid=0 2024-11-25T03:39:27,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742332_1508 (size=14594) 2024-11-25T03:39:27,174 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:27,178 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125dc72c12b206644a6957a02dd15314092_33de131a28520cc9ecc2bf159dfd6128 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125dc72c12b206644a6957a02dd15314092_33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:27,178 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/37f8a4d8e85d4da9bae5d50c5e33193d, store: [table=TestAcidGuarantees family=A region=33de131a28520cc9ecc2bf159dfd6128] 2024-11-25T03:39:27,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/37f8a4d8e85d4da9bae5d50c5e33193d is 175, key is test_row_0/A:col10/1732505967158/Put/seqid=0 2024-11-25T03:39:27,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742333_1509 (size=39549) 2024-11-25T03:39:27,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:27,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39858 deadline: 1732506027225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:27,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:27,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:27,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:27,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:27,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1732506027225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:27,230 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:27,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:27,230 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:27,230 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:27,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39884 deadline: 1732506027226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:27,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:27,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:27,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:27,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:27,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39858 deadline: 1732506027337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:27,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:27,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39884 deadline: 1732506027337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:27,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:27,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:27,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:27,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:27,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:27,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:27,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1732506027337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:27,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:27,340 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:27,340 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:27,340 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:27,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:27,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39858 deadline: 1732506027548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:27,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:27,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:27,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39884 deadline: 1732506027548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:27,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:27,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:27,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:27,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:27,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1732506027548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:27,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:27,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:27,550 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:27,550 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:27,550 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:27,584 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=55, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/37f8a4d8e85d4da9bae5d50c5e33193d 2024-11-25T03:39:27,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/370b7b0db6414865bb62c81e694ce820 is 50, key is test_row_0/B:col10/1732505967158/Put/seqid=0 2024-11-25T03:39:27,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742334_1510 (size=12001) 2024-11-25T03:39:27,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:27,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39858 deadline: 1732506027857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:27,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:27,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:27,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:27,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39884 deadline: 1732506027857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:27,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:27,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:27,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:27,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:27,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:27,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1732506027857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:27,861 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:27,861 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:27,861 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:28,007 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/370b7b0db6414865bb62c81e694ce820 2024-11-25T03:39:28,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/7ec138df5c644e4f80ad22f23e0299f7 is 50, key is test_row_0/C:col10/1732505967158/Put/seqid=0 2024-11-25T03:39:28,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742335_1511 (size=12001) 2024-11-25T03:39:28,018 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/7ec138df5c644e4f80ad22f23e0299f7 2024-11-25T03:39:28,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/37f8a4d8e85d4da9bae5d50c5e33193d as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/37f8a4d8e85d4da9bae5d50c5e33193d 2024-11-25T03:39:28,024 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/37f8a4d8e85d4da9bae5d50c5e33193d, entries=200, sequenceid=55, filesize=38.6 K 2024-11-25T03:39:28,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/370b7b0db6414865bb62c81e694ce820 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/370b7b0db6414865bb62c81e694ce820 2024-11-25T03:39:28,028 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/370b7b0db6414865bb62c81e694ce820, entries=150, sequenceid=55, filesize=11.7 K 2024-11-25T03:39:28,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/7ec138df5c644e4f80ad22f23e0299f7 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/7ec138df5c644e4f80ad22f23e0299f7 2024-11-25T03:39:28,032 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/7ec138df5c644e4f80ad22f23e0299f7, entries=150, sequenceid=55, filesize=11.7 K 2024-11-25T03:39:28,032 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=140.89 KB/144270 for 33de131a28520cc9ecc2bf159dfd6128 in 872ms, sequenceid=55, compaction requested=true 2024-11-25T03:39:28,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 33de131a28520cc9ecc2bf159dfd6128: 2024-11-25T03:39:28,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33de131a28520cc9ecc2bf159dfd6128:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:39:28,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:39:28,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33de131a28520cc9ecc2bf159dfd6128:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:39:28,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:39:28,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33de131a28520cc9ecc2bf159dfd6128:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:39:28,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:39:28,033 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:39:28,033 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:39:28,033 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:39:28,033 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:39:28,033 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 33de131a28520cc9ecc2bf159dfd6128/B is initiating minor compaction (all files) 2024-11-25T03:39:28,033 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): 33de131a28520cc9ecc2bf159dfd6128/A is initiating minor compaction (all files) 2024-11-25T03:39:28,034 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33de131a28520cc9ecc2bf159dfd6128/B in TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:28,034 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33de131a28520cc9ecc2bf159dfd6128/A in TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:28,034 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/d154319122014d238fc3a0508af7c668, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/b4c6ba40e98f4b6f94c8974a07cc1198, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/370b7b0db6414865bb62c81e694ce820] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp, totalSize=35.2 K 2024-11-25T03:39:28,034 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/6ad3d64a9c404233b1730e18f65d2024, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/7f1037a2539447409d0156d2b5b4d7a9, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/37f8a4d8e85d4da9bae5d50c5e33193d] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp, totalSize=107.5 K 2024-11-25T03:39:28,034 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:28,034 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. files: [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/6ad3d64a9c404233b1730e18f65d2024, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/7f1037a2539447409d0156d2b5b4d7a9, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/37f8a4d8e85d4da9bae5d50c5e33193d] 2024-11-25T03:39:28,034 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting d154319122014d238fc3a0508af7c668, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732505964858 2024-11-25T03:39:28,034 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ad3d64a9c404233b1730e18f65d2024, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732505964858 2024-11-25T03:39:28,034 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting b4c6ba40e98f4b6f94c8974a07cc1198, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732505964878 2024-11-25T03:39:28,035 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f1037a2539447409d0156d2b5b4d7a9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732505964878 2024-11-25T03:39:28,035 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 370b7b0db6414865bb62c81e694ce820, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732505967158 2024-11-25T03:39:28,035 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37f8a4d8e85d4da9bae5d50c5e33193d, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732505967038 2024-11-25T03:39:28,041 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33de131a28520cc9ecc2bf159dfd6128#B#compaction#434 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:39:28,042 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/0aac5e69dbfc456e90d83c14af3dfb28 is 50, key is test_row_0/B:col10/1732505967158/Put/seqid=0 2024-11-25T03:39:28,049 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=33de131a28520cc9ecc2bf159dfd6128] 2024-11-25T03:39:28,055 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112524870d81482b41fdbfbcab1e14c782fd_33de131a28520cc9ecc2bf159dfd6128 store=[table=TestAcidGuarantees family=A region=33de131a28520cc9ecc2bf159dfd6128] 2024-11-25T03:39:28,057 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112524870d81482b41fdbfbcab1e14c782fd_33de131a28520cc9ecc2bf159dfd6128, store=[table=TestAcidGuarantees family=A region=33de131a28520cc9ecc2bf159dfd6128] 2024-11-25T03:39:28,057 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112524870d81482b41fdbfbcab1e14c782fd_33de131a28520cc9ecc2bf159dfd6128 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=33de131a28520cc9ecc2bf159dfd6128] 2024-11-25T03:39:28,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742336_1512 (size=12104) 2024-11-25T03:39:28,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742337_1513 (size=4469) 2024-11-25T03:39:28,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:28,369 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:39:28,369 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 33de131a28520cc9ecc2bf159dfd6128 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-25T03:39:28,377 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411255aa8c766a3be448aa56fb93d7ca88509_33de131a28520cc9ecc2bf159dfd6128 is 50, key is test_row_0/A:col10/1732505967224/Put/seqid=0 2024-11-25T03:39:28,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742338_1514 (size=14594) 2024-11-25T03:39:28,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:28,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39858 deadline: 1732506028379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:28,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:28,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:28,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:28,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:28,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1732506028383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:28,387 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:28,387 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:28,387 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:28,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:28,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39884 deadline: 1732506028384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:28,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:28,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:28,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:28,470 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/0aac5e69dbfc456e90d83c14af3dfb28 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/0aac5e69dbfc456e90d83c14af3dfb28 2024-11-25T03:39:28,471 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33de131a28520cc9ecc2bf159dfd6128#A#compaction#435 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:39:28,472 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/7a2c3e4403304167b467fb0e0d5fa474 is 175, key is test_row_0/A:col10/1732505967158/Put/seqid=0 2024-11-25T03:39:28,474 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 33de131a28520cc9ecc2bf159dfd6128/B of 33de131a28520cc9ecc2bf159dfd6128 into 0aac5e69dbfc456e90d83c14af3dfb28(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:39:28,474 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33de131a28520cc9ecc2bf159dfd6128: 2024-11-25T03:39:28,474 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., storeName=33de131a28520cc9ecc2bf159dfd6128/B, priority=13, startTime=1732505968033; duration=0sec 2024-11-25T03:39:28,475 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:39:28,475 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33de131a28520cc9ecc2bf159dfd6128:B 2024-11-25T03:39:28,475 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:39:28,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742339_1515 (size=31058) 2024-11-25T03:39:28,476 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:39:28,476 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 33de131a28520cc9ecc2bf159dfd6128/C is initiating minor compaction (all files) 2024-11-25T03:39:28,476 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33de131a28520cc9ecc2bf159dfd6128/C in TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:28,476 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/fe66b944adce47ee9dd2530b2fec1106, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/c6d87f70a45e42a48877552edc5687d3, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/7ec138df5c644e4f80ad22f23e0299f7] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp, totalSize=35.2 K 2024-11-25T03:39:28,477 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting fe66b944adce47ee9dd2530b2fec1106, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732505964858 2024-11-25T03:39:28,477 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting c6d87f70a45e42a48877552edc5687d3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732505964878 2024-11-25T03:39:28,477 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ec138df5c644e4f80ad22f23e0299f7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732505967158 2024-11-25T03:39:28,482 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33de131a28520cc9ecc2bf159dfd6128#C#compaction#437 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:39:28,483 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/b37f2e9ed0bd4a7b8f0352a0bf29ad00 is 50, key is test_row_0/C:col10/1732505967158/Put/seqid=0 2024-11-25T03:39:28,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742340_1516 (size=12104) 2024-11-25T03:39:28,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:28,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39858 deadline: 1732506028487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:28,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:28,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:28,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:28,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:28,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1732506028498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:28,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:28,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39884 deadline: 1732506028498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:28,501 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:28,501 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:28,501 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:28,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:28,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:28,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:28,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:28,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39858 deadline: 1732506028697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:28,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:28,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:28,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:28,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:28,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1732506028708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:28,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:28,710 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:28,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39884 deadline: 1732506028708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:28,710 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:28,710 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:28,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:28,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:28,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:28,782 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:28,785 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411255aa8c766a3be448aa56fb93d7ca88509_33de131a28520cc9ecc2bf159dfd6128 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411255aa8c766a3be448aa56fb93d7ca88509_33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:28,786 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/76696f1b1d944ec79bffafc7767981c5, store: [table=TestAcidGuarantees family=A region=33de131a28520cc9ecc2bf159dfd6128] 2024-11-25T03:39:28,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/76696f1b1d944ec79bffafc7767981c5 is 175, key is test_row_0/A:col10/1732505967224/Put/seqid=0 2024-11-25T03:39:28,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742341_1517 (size=39549) 2024-11-25T03:39:28,880 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/7a2c3e4403304167b467fb0e0d5fa474 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/7a2c3e4403304167b467fb0e0d5fa474 2024-11-25T03:39:28,884 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 33de131a28520cc9ecc2bf159dfd6128/A of 33de131a28520cc9ecc2bf159dfd6128 into 7a2c3e4403304167b467fb0e0d5fa474(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:39:28,884 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33de131a28520cc9ecc2bf159dfd6128: 2024-11-25T03:39:28,884 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., storeName=33de131a28520cc9ecc2bf159dfd6128/A, priority=13, startTime=1732505968032; duration=0sec 2024-11-25T03:39:28,884 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:39:28,884 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33de131a28520cc9ecc2bf159dfd6128:A 2024-11-25T03:39:28,892 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/b37f2e9ed0bd4a7b8f0352a0bf29ad00 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/b37f2e9ed0bd4a7b8f0352a0bf29ad00 2024-11-25T03:39:28,896 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 33de131a28520cc9ecc2bf159dfd6128/C of 33de131a28520cc9ecc2bf159dfd6128 into b37f2e9ed0bd4a7b8f0352a0bf29ad00(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:39:28,896 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33de131a28520cc9ecc2bf159dfd6128: 2024-11-25T03:39:28,896 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., storeName=33de131a28520cc9ecc2bf159dfd6128/C, priority=13, startTime=1732505968033; duration=0sec 2024-11-25T03:39:28,896 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:39:28,896 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33de131a28520cc9ecc2bf159dfd6128:C 2024-11-25T03:39:29,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:29,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39858 deadline: 1732506029007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:29,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:29,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:29,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:29,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-25T03:39:29,017 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:39:29,018 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:39:29,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-25T03:39:29,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-25T03:39:29,020 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:39:29,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:29,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1732506029017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:29,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:29,021 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:39:29,021 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:29,021 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:29,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39884 deadline: 1732506029017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:29,021 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:29,021 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:39:29,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:29,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:29,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:29,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:29,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1732506029067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:29,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:29,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39900 deadline: 1732506029067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:29,071 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,34881,1732505806588 for 'test_row_0' in 33de131a28520cc9ecc2bf159dfd6128 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:39:29,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:29,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:29,071 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,34881,1732505806588 for 'test_row_2' in 33de131a28520cc9ecc2bf159dfd6128 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:39:29,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:29,071 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:29,071 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:29,071 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:29,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-25T03:39:29,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-25T03:39:29,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:29,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. as already flushing 2024-11-25T03:39:29,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:29,173 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:29,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:29,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:29,190 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=81, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/76696f1b1d944ec79bffafc7767981c5 2024-11-25T03:39:29,197 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/41729ae2ea664b0f8333b936edd5991e is 50, key is test_row_0/B:col10/1732505967224/Put/seqid=0 2024-11-25T03:39:29,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742342_1518 (size=12001) 2024-11-25T03:39:29,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-25T03:39:29,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:29,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. as already flushing 2024-11-25T03:39:29,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:29,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:29,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:29,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:29,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-25T03:39:29,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-25T03:39:29,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:29,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. as already flushing 2024-11-25T03:39:29,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:29,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:29,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:29,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:29,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:29,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39858 deadline: 1732506029517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:29,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:29,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:29,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:29,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:29,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1732506029528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:29,530 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:29,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:29,530 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:29,530 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:29,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39884 deadline: 1732506029528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:29,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:29,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:29,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:29,601 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/41729ae2ea664b0f8333b936edd5991e 2024-11-25T03:39:29,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/c861c53dbd2541feb7352d641b84d007 is 50, key is test_row_0/C:col10/1732505967224/Put/seqid=0 2024-11-25T03:39:29,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742343_1519 (size=12001) 2024-11-25T03:39:29,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-25T03:39:29,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:29,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. as already flushing 2024-11-25T03:39:29,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:29,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:29,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:29,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:29,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-25T03:39:29,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-25T03:39:29,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:29,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. as already flushing 2024-11-25T03:39:29,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:29,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:29,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:29,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:29,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-25T03:39:29,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:29,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. as already flushing 2024-11-25T03:39:29,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:29,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:29,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:29,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:30,012 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/c861c53dbd2541feb7352d641b84d007 2024-11-25T03:39:30,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/76696f1b1d944ec79bffafc7767981c5 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/76696f1b1d944ec79bffafc7767981c5 2024-11-25T03:39:30,018 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/76696f1b1d944ec79bffafc7767981c5, entries=200, sequenceid=81, filesize=38.6 K 2024-11-25T03:39:30,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/41729ae2ea664b0f8333b936edd5991e as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/41729ae2ea664b0f8333b936edd5991e 2024-11-25T03:39:30,022 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/41729ae2ea664b0f8333b936edd5991e, entries=150, sequenceid=81, filesize=11.7 K 2024-11-25T03:39:30,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/c861c53dbd2541feb7352d641b84d007 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/c861c53dbd2541feb7352d641b84d007 2024-11-25T03:39:30,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/c861c53dbd2541feb7352d641b84d007, entries=150, sequenceid=81, filesize=11.7 K 2024-11-25T03:39:30,027 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 33de131a28520cc9ecc2bf159dfd6128 in 1658ms, sequenceid=81, compaction requested=false 2024-11-25T03:39:30,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 33de131a28520cc9ecc2bf159dfd6128: 2024-11-25T03:39:30,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-25T03:39:30,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:30,091 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 33de131a28520cc9ecc2bf159dfd6128 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-25T03:39:30,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112585db22c7b31840c486c0f140823bd74d_33de131a28520cc9ecc2bf159dfd6128 is 50, key is test_row_0/A:col10/1732505968377/Put/seqid=0 2024-11-25T03:39:30,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742344_1520 (size=12154) 2024-11-25T03:39:30,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-25T03:39:30,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:30,508 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112585db22c7b31840c486c0f140823bd74d_33de131a28520cc9ecc2bf159dfd6128 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112585db22c7b31840c486c0f140823bd74d_33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:30,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/0d2e09f840fc427b8dcaad02fb273330, store: [table=TestAcidGuarantees family=A region=33de131a28520cc9ecc2bf159dfd6128] 2024-11-25T03:39:30,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/0d2e09f840fc427b8dcaad02fb273330 is 175, key is test_row_0/A:col10/1732505968377/Put/seqid=0 2024-11-25T03:39:30,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742345_1521 (size=30955) 2024-11-25T03:39:30,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. as already flushing 2024-11-25T03:39:30,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:30,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:30,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39884 deadline: 1732506030577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:30,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:30,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:30,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:30,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:30,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39858 deadline: 1732506030578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:30,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:30,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:30,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:30,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:30,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1732506030579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:30,583 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:30,584 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:30,584 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:30,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:30,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39884 deadline: 1732506030687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:30,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:30,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:30,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:30,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:30,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39858 deadline: 1732506030688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:30,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:30,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:30,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:30,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:30,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1732506030688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:30,692 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:30,692 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:30,692 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:30,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:30,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39884 deadline: 1732506030897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:30,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:30,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:30,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:30,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:30,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1732506030898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:30,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:30,901 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:30,902 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:30,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39858 deadline: 1732506030898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:30,902 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:30,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:30,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:30,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:30,928 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=94, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/0d2e09f840fc427b8dcaad02fb273330 2024-11-25T03:39:30,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/07242c03c4b64266aeb21c0202339dfe is 50, key is test_row_0/B:col10/1732505968377/Put/seqid=0 2024-11-25T03:39:30,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742346_1522 (size=12001) 2024-11-25T03:39:30,963 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/07242c03c4b64266aeb21c0202339dfe 2024-11-25T03:39:30,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/6976775bf9884734a52a28d750e904c2 is 50, key is test_row_0/C:col10/1732505968377/Put/seqid=0 2024-11-25T03:39:30,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742347_1523 (size=12001) 2024-11-25T03:39:31,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-25T03:39:31,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:31,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39884 deadline: 1732506031207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:31,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:31,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:31,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1732506031208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:31,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:31,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:31,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:31,211 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:31,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39858 deadline: 1732506031208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:31,211 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:31,211 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:31,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:31,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:31,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:31,399 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/6976775bf9884734a52a28d750e904c2 2024-11-25T03:39:31,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/0d2e09f840fc427b8dcaad02fb273330 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/0d2e09f840fc427b8dcaad02fb273330 2024-11-25T03:39:31,408 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/0d2e09f840fc427b8dcaad02fb273330, entries=150, sequenceid=94, filesize=30.2 K 2024-11-25T03:39:31,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/07242c03c4b64266aeb21c0202339dfe as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/07242c03c4b64266aeb21c0202339dfe 2024-11-25T03:39:31,414 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/07242c03c4b64266aeb21c0202339dfe, entries=150, sequenceid=94, filesize=11.7 K 2024-11-25T03:39:31,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/6976775bf9884734a52a28d750e904c2 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/6976775bf9884734a52a28d750e904c2 2024-11-25T03:39:31,425 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/6976775bf9884734a52a28d750e904c2, entries=150, sequenceid=94, filesize=11.7 K 2024-11-25T03:39:31,426 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 33de131a28520cc9ecc2bf159dfd6128 in 1336ms, sequenceid=94, compaction requested=true 2024-11-25T03:39:31,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 33de131a28520cc9ecc2bf159dfd6128: 2024-11-25T03:39:31,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:31,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-25T03:39:31,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-25T03:39:31,430 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-25T03:39:31,430 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4070 sec 2024-11-25T03:39:31,433 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 2.4130 sec 2024-11-25T03:39:31,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:31,720 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:39:31,720 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 33de131a28520cc9ecc2bf159dfd6128 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-25T03:39:31,727 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125003e24526fb5428688a7b543fa329882_33de131a28520cc9ecc2bf159dfd6128 is 50, key is test_row_0/A:col10/1732505970576/Put/seqid=0 2024-11-25T03:39:31,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742348_1524 (size=12154) 2024-11-25T03:39:31,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4773) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4699) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5194) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5188) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5184) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3333) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:31,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:31,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1732506031727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:31,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39858 deadline: 1732506031728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:31,732 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4773) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4699) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5194) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5188) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5184) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3333) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:31,732 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4773) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4699) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5194) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5188) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5184) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3333) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:31,732 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:31,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:31,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:31,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:31,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:31,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39884 deadline: 1732506031732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:31,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:31,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:31,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:31,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:31,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1732506031837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:31,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:31,838 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:31,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39858 deadline: 1732506031837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:31,838 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:31,838 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:31,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:31,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:31,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:31,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39884 deadline: 1732506031838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:31,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:31,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:31,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:31,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:32,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:32,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1732506032047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:32,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:32,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39858 deadline: 1732506032047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:32,048 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:32,048 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:32,048 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:32,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:32,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39884 deadline: 1732506032047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:32,131 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:32,134 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125003e24526fb5428688a7b543fa329882_33de131a28520cc9ecc2bf159dfd6128 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125003e24526fb5428688a7b543fa329882_33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:32,135 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/1dc11343b8cb40f29142a0ffb3ffc46f, store: [table=TestAcidGuarantees family=A region=33de131a28520cc9ecc2bf159dfd6128] 2024-11-25T03:39:32,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/1dc11343b8cb40f29142a0ffb3ffc46f is 175, key is test_row_0/A:col10/1732505970576/Put/seqid=0 2024-11-25T03:39:32,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742349_1525 (size=30955) 2024-11-25T03:39:32,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:32,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1732506032357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:32,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:32,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:32,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39858 deadline: 1732506032357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:32,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39884 deadline: 1732506032357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:32,362 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:32,362 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:32,362 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:32,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:32,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:32,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:32,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:32,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:32,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:32,540 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=122, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/1dc11343b8cb40f29142a0ffb3ffc46f 2024-11-25T03:39:32,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/34e47fb9708546c4acd1b55de485897b is 50, key is test_row_0/B:col10/1732505970576/Put/seqid=0 2024-11-25T03:39:32,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742350_1526 (size=12001) 2024-11-25T03:39:32,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:32,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1732506032867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:32,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:32,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39858 deadline: 1732506032867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:32,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:32,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:32,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39884 deadline: 1732506032867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:32,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:32,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:32,871 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:32,871 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:32,871 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:32,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:32,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:32,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:32,952 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/34e47fb9708546c4acd1b55de485897b 2024-11-25T03:39:32,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/9017ce44b4364640b1eddc38b54c8738 is 50, key is test_row_0/C:col10/1732505970576/Put/seqid=0 2024-11-25T03:39:32,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742351_1527 (size=12001) 2024-11-25T03:39:32,970 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/9017ce44b4364640b1eddc38b54c8738 2024-11-25T03:39:32,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/1dc11343b8cb40f29142a0ffb3ffc46f as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/1dc11343b8cb40f29142a0ffb3ffc46f 2024-11-25T03:39:32,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/1dc11343b8cb40f29142a0ffb3ffc46f, entries=150, sequenceid=122, filesize=30.2 K 2024-11-25T03:39:32,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/34e47fb9708546c4acd1b55de485897b as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/34e47fb9708546c4acd1b55de485897b 2024-11-25T03:39:32,981 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/34e47fb9708546c4acd1b55de485897b, entries=150, sequenceid=122, filesize=11.7 K 2024-11-25T03:39:32,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/9017ce44b4364640b1eddc38b54c8738 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/9017ce44b4364640b1eddc38b54c8738 2024-11-25T03:39:32,986 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/9017ce44b4364640b1eddc38b54c8738, entries=150, sequenceid=122, filesize=11.7 K 2024-11-25T03:39:32,986 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 33de131a28520cc9ecc2bf159dfd6128 in 1266ms, sequenceid=122, compaction requested=true 2024-11-25T03:39:32,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 33de131a28520cc9ecc2bf159dfd6128: 2024-11-25T03:39:32,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33de131a28520cc9ecc2bf159dfd6128:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:39:32,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:39:32,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33de131a28520cc9ecc2bf159dfd6128:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:39:32,987 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:39:32,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:39:32,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33de131a28520cc9ecc2bf159dfd6128:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:39:32,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T03:39:32,987 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:39:32,989 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:39:32,989 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132517 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:39:32,989 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): 33de131a28520cc9ecc2bf159dfd6128/A is initiating minor compaction (all files) 2024-11-25T03:39:32,989 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 33de131a28520cc9ecc2bf159dfd6128/B is initiating minor compaction (all files) 2024-11-25T03:39:32,989 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33de131a28520cc9ecc2bf159dfd6128/A in TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:32,989 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33de131a28520cc9ecc2bf159dfd6128/B in TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:32,989 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/7a2c3e4403304167b467fb0e0d5fa474, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/76696f1b1d944ec79bffafc7767981c5, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/0d2e09f840fc427b8dcaad02fb273330, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/1dc11343b8cb40f29142a0ffb3ffc46f] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp, totalSize=129.4 K 2024-11-25T03:39:32,989 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/0aac5e69dbfc456e90d83c14af3dfb28, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/41729ae2ea664b0f8333b936edd5991e, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/07242c03c4b64266aeb21c0202339dfe, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/34e47fb9708546c4acd1b55de485897b] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp, totalSize=47.0 K 2024-11-25T03:39:32,989 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:32,989 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. files: [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/7a2c3e4403304167b467fb0e0d5fa474, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/76696f1b1d944ec79bffafc7767981c5, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/0d2e09f840fc427b8dcaad02fb273330, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/1dc11343b8cb40f29142a0ffb3ffc46f] 2024-11-25T03:39:32,990 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 0aac5e69dbfc456e90d83c14af3dfb28, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732505967158 2024-11-25T03:39:32,990 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a2c3e4403304167b467fb0e0d5fa474, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732505967158 2024-11-25T03:39:32,990 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76696f1b1d944ec79bffafc7767981c5, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732505967223 2024-11-25T03:39:32,990 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 41729ae2ea664b0f8333b936edd5991e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732505967224 2024-11-25T03:39:32,991 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d2e09f840fc427b8dcaad02fb273330, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732505968377 2024-11-25T03:39:32,991 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 07242c03c4b64266aeb21c0202339dfe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732505968377 2024-11-25T03:39:32,992 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1dc11343b8cb40f29142a0ffb3ffc46f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732505970576 2024-11-25T03:39:32,993 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 34e47fb9708546c4acd1b55de485897b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732505970576 2024-11-25T03:39:33,000 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=33de131a28520cc9ecc2bf159dfd6128] 2024-11-25T03:39:33,002 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125129996a9dff048ec928c178756fa351d_33de131a28520cc9ecc2bf159dfd6128 store=[table=TestAcidGuarantees family=A region=33de131a28520cc9ecc2bf159dfd6128] 2024-11-25T03:39:33,005 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125129996a9dff048ec928c178756fa351d_33de131a28520cc9ecc2bf159dfd6128, store=[table=TestAcidGuarantees family=A region=33de131a28520cc9ecc2bf159dfd6128] 2024-11-25T03:39:33,005 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125129996a9dff048ec928c178756fa351d_33de131a28520cc9ecc2bf159dfd6128 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=33de131a28520cc9ecc2bf159dfd6128] 2024-11-25T03:39:33,005 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33de131a28520cc9ecc2bf159dfd6128#B#compaction#447 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:39:33,006 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/2835373f0ea3427b9477659423dfbdc5 is 50, key is test_row_0/B:col10/1732505970576/Put/seqid=0 2024-11-25T03:39:33,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742353_1529 (size=12241) 2024-11-25T03:39:33,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742352_1528 (size=4469) 2024-11-25T03:39:33,018 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/2835373f0ea3427b9477659423dfbdc5 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/2835373f0ea3427b9477659423dfbdc5 2024-11-25T03:39:33,022 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 33de131a28520cc9ecc2bf159dfd6128/B of 33de131a28520cc9ecc2bf159dfd6128 into 2835373f0ea3427b9477659423dfbdc5(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:39:33,023 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33de131a28520cc9ecc2bf159dfd6128: 2024-11-25T03:39:33,023 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., storeName=33de131a28520cc9ecc2bf159dfd6128/B, priority=12, startTime=1732505972987; duration=0sec 2024-11-25T03:39:33,023 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:39:33,023 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33de131a28520cc9ecc2bf159dfd6128:B 2024-11-25T03:39:33,023 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:39:33,024 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:39:33,024 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 33de131a28520cc9ecc2bf159dfd6128/C is initiating minor compaction (all files) 2024-11-25T03:39:33,024 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33de131a28520cc9ecc2bf159dfd6128/C in TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:33,025 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/b37f2e9ed0bd4a7b8f0352a0bf29ad00, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/c861c53dbd2541feb7352d641b84d007, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/6976775bf9884734a52a28d750e904c2, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/9017ce44b4364640b1eddc38b54c8738] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp, totalSize=47.0 K 2024-11-25T03:39:33,025 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting b37f2e9ed0bd4a7b8f0352a0bf29ad00, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732505967158 2024-11-25T03:39:33,025 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting c861c53dbd2541feb7352d641b84d007, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732505967224 2024-11-25T03:39:33,025 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 6976775bf9884734a52a28d750e904c2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732505968377 2024-11-25T03:39:33,025 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 9017ce44b4364640b1eddc38b54c8738, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732505970576 2024-11-25T03:39:33,032 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33de131a28520cc9ecc2bf159dfd6128#C#compaction#448 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:39:33,032 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/38deacbe2872454ba52d56e687e8e3f3 is 50, key is test_row_0/C:col10/1732505970576/Put/seqid=0 2024-11-25T03:39:33,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742354_1530 (size=12241) 2024-11-25T03:39:33,039 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/38deacbe2872454ba52d56e687e8e3f3 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/38deacbe2872454ba52d56e687e8e3f3 2024-11-25T03:39:33,044 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 33de131a28520cc9ecc2bf159dfd6128/C of 33de131a28520cc9ecc2bf159dfd6128 into 38deacbe2872454ba52d56e687e8e3f3(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:39:33,044 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33de131a28520cc9ecc2bf159dfd6128: 2024-11-25T03:39:33,044 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., storeName=33de131a28520cc9ecc2bf159dfd6128/C, priority=12, startTime=1732505972987; duration=0sec 2024-11-25T03:39:33,044 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:39:33,044 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33de131a28520cc9ecc2bf159dfd6128:C 2024-11-25T03:39:33,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:33,095 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:39:33,095 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 33de131a28520cc9ecc2bf159dfd6128 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-25T03:39:33,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112570f37f2a9ed04efba5dd580e64293bfb_33de131a28520cc9ecc2bf159dfd6128 is 50, key is test_row_0/A:col10/1732505973091/Put/seqid=0 2024-11-25T03:39:33,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742355_1531 (size=14694) 2024-11-25T03:39:33,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:33,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39900 deadline: 1732506033158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:33,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:33,163 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:33,163 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:33,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1732506033159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:33,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:33,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:33,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:33,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:33,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-25T03:39:33,187 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:39:33,187 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:39:33,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-25T03:39:33,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-25T03:39:33,189 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:39:33,190 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:39:33,190 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:39:33,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:33,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39900 deadline: 1732506033268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:33,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:33,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:33,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:33,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1732506033268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:33,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:33,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:33,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:33,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:33,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-25T03:39:33,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-25T03:39:33,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:33,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. as already flushing 2024-11-25T03:39:33,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:33,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:33,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:33,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:33,416 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33de131a28520cc9ecc2bf159dfd6128#A#compaction#446 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:39:33,416 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/52100752fcf04c22bc0cad7795bbefe8 is 175, key is test_row_0/A:col10/1732505970576/Put/seqid=0 2024-11-25T03:39:33,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742356_1532 (size=31195) 2024-11-25T03:39:33,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:33,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39900 deadline: 1732506033477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:33,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:33,481 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:33,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1732506033478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:33,481 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:33,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:33,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:33,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:33,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:33,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-25T03:39:33,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:33,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. as already flushing 2024-11-25T03:39:33,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:33,495 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:33,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:33,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:33,505 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:33,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-25T03:39:33,509 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112570f37f2a9ed04efba5dd580e64293bfb_33de131a28520cc9ecc2bf159dfd6128 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112570f37f2a9ed04efba5dd580e64293bfb_33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:33,509 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/15e329e39f2b4f41bb7befe87e6e17b6, store: [table=TestAcidGuarantees family=A region=33de131a28520cc9ecc2bf159dfd6128] 2024-11-25T03:39:33,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/15e329e39f2b4f41bb7befe87e6e17b6 is 175, key is test_row_0/A:col10/1732505973091/Put/seqid=0 2024-11-25T03:39:33,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742357_1533 (size=39649) 2024-11-25T03:39:33,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-25T03:39:33,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:33,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. as already flushing 2024-11-25T03:39:33,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:33,648 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:33,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:33,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:33,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:33,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39900 deadline: 1732506033787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:33,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:33,789 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:33,789 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:33,789 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:33,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1732506033787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:33,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:33,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:33,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:33,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-25T03:39:33,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:33,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. as already flushing 2024-11-25T03:39:33,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:33,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:33,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:33,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:33,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-25T03:39:33,824 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/52100752fcf04c22bc0cad7795bbefe8 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/52100752fcf04c22bc0cad7795bbefe8 2024-11-25T03:39:33,828 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 33de131a28520cc9ecc2bf159dfd6128/A of 33de131a28520cc9ecc2bf159dfd6128 into 52100752fcf04c22bc0cad7795bbefe8(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:39:33,828 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33de131a28520cc9ecc2bf159dfd6128: 2024-11-25T03:39:33,828 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., storeName=33de131a28520cc9ecc2bf159dfd6128/A, priority=12, startTime=1732505972987; duration=0sec 2024-11-25T03:39:33,828 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:39:33,828 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33de131a28520cc9ecc2bf159dfd6128:A 2024-11-25T03:39:33,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:33,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39858 deadline: 1732506033887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:33,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:33,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1732506033887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:33,890 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,34881,1732505806588 for 'test_row_2' in 33de131a28520cc9ecc2bf159dfd6128 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:39:33,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:33,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:33,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39884 deadline: 1732506033887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:33,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:33,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:33,890 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,34881,1732505806588 for 'test_row_0' in 33de131a28520cc9ecc2bf159dfd6128 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:39:33,891 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:33,891 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:33,891 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:33,891 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,34881,1732505806588 for 'test_row_0' in 33de131a28520cc9ecc2bf159dfd6128 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:39:33,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:33,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:33,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:33,914 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=135, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/15e329e39f2b4f41bb7befe87e6e17b6 2024-11-25T03:39:33,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/6f831653fa694f90b2e469d85fee9dc4 is 50, key is test_row_0/B:col10/1732505973091/Put/seqid=0 2024-11-25T03:39:33,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742358_1534 (size=12101) 2024-11-25T03:39:33,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-25T03:39:33,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:33,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. as already flushing 2024-11-25T03:39:33,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:33,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:33,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:33,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:34,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-25T03:39:34,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:34,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. as already flushing 2024-11-25T03:39:34,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:34,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:34,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:34,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:34,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-25T03:39:34,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:34,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. as already flushing 2024-11-25T03:39:34,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:34,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:34,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:34,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:34,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:34,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39900 deadline: 1732506034297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:34,300 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:34,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:34,301 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:34,301 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:34,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1732506034298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:34,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:34,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:34,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:34,323 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/6f831653fa694f90b2e469d85fee9dc4 2024-11-25T03:39:34,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-25T03:39:34,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/593f2d6d4452449ab090f748a93b917a is 50, key is test_row_0/C:col10/1732505973091/Put/seqid=0 2024-11-25T03:39:34,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742359_1535 (size=12101) 2024-11-25T03:39:34,333 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/593f2d6d4452449ab090f748a93b917a 2024-11-25T03:39:34,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/15e329e39f2b4f41bb7befe87e6e17b6 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/15e329e39f2b4f41bb7befe87e6e17b6 2024-11-25T03:39:34,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/15e329e39f2b4f41bb7befe87e6e17b6, entries=200, sequenceid=135, filesize=38.7 K 2024-11-25T03:39:34,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/6f831653fa694f90b2e469d85fee9dc4 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/6f831653fa694f90b2e469d85fee9dc4 2024-11-25T03:39:34,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/6f831653fa694f90b2e469d85fee9dc4, entries=150, sequenceid=135, filesize=11.8 K 2024-11-25T03:39:34,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/593f2d6d4452449ab090f748a93b917a as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/593f2d6d4452449ab090f748a93b917a 2024-11-25T03:39:34,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/593f2d6d4452449ab090f748a93b917a, entries=150, sequenceid=135, filesize=11.8 K 2024-11-25T03:39:34,348 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 33de131a28520cc9ecc2bf159dfd6128 in 1253ms, sequenceid=135, compaction requested=false 2024-11-25T03:39:34,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 33de131a28520cc9ecc2bf159dfd6128: 2024-11-25T03:39:34,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-25T03:39:34,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:34,413 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 33de131a28520cc9ecc2bf159dfd6128 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-25T03:39:34,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112508f28a9268044bac92570f0c47fedbef_33de131a28520cc9ecc2bf159dfd6128 is 50, key is test_row_0/A:col10/1732505973143/Put/seqid=0 2024-11-25T03:39:34,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742360_1536 (size=12304) 2024-11-25T03:39:34,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:34,428 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112508f28a9268044bac92570f0c47fedbef_33de131a28520cc9ecc2bf159dfd6128 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112508f28a9268044bac92570f0c47fedbef_33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:34,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/98b96364bccd453197ee21c6a15b980f, store: [table=TestAcidGuarantees family=A region=33de131a28520cc9ecc2bf159dfd6128] 2024-11-25T03:39:34,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/98b96364bccd453197ee21c6a15b980f is 175, key is test_row_0/A:col10/1732505973143/Put/seqid=0 2024-11-25T03:39:34,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742361_1537 (size=31105) 2024-11-25T03:39:34,433 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=161, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/98b96364bccd453197ee21c6a15b980f 2024-11-25T03:39:34,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/26724993e6b9468d938aac8e6bbe5f56 is 50, key is test_row_0/B:col10/1732505973143/Put/seqid=0 2024-11-25T03:39:34,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742362_1538 (size=12151) 2024-11-25T03:39:34,845 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/26724993e6b9468d938aac8e6bbe5f56 2024-11-25T03:39:34,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/4fcae415bc81404d823b80aa7f0cd78a is 50, key is test_row_0/C:col10/1732505973143/Put/seqid=0 2024-11-25T03:39:34,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742363_1539 (size=12151) 2024-11-25T03:39:35,257 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/4fcae415bc81404d823b80aa7f0cd78a 2024-11-25T03:39:35,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/98b96364bccd453197ee21c6a15b980f as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/98b96364bccd453197ee21c6a15b980f 2024-11-25T03:39:35,264 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/98b96364bccd453197ee21c6a15b980f, entries=150, sequenceid=161, filesize=30.4 K 2024-11-25T03:39:35,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/26724993e6b9468d938aac8e6bbe5f56 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/26724993e6b9468d938aac8e6bbe5f56 2024-11-25T03:39:35,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,267 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/26724993e6b9468d938aac8e6bbe5f56, entries=150, sequenceid=161, filesize=11.9 K 2024-11-25T03:39:35,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/4fcae415bc81404d823b80aa7f0cd78a as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/4fcae415bc81404d823b80aa7f0cd78a 2024-11-25T03:39:35,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,272 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/4fcae415bc81404d823b80aa7f0cd78a, entries=150, sequenceid=161, filesize=11.9 K 2024-11-25T03:39:35,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,273 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for 33de131a28520cc9ecc2bf159dfd6128 in 859ms, sequenceid=161, compaction requested=true 2024-11-25T03:39:35,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 33de131a28520cc9ecc2bf159dfd6128: 2024-11-25T03:39:35,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:35,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-25T03:39:35,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-25T03:39:35,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,276 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-25T03:39:35,276 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0840 sec 2024-11-25T03:39:35,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,278 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 2.0890 sec 2024-11-25T03:39:35,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-25T03:39:35,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,338 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:39:35,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,338 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:39:35,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-25T03:39:35,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-25T03:39:35,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,340 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:39:35,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,341 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:39:35,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,341 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:39:35,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:35,368 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:39:35,368 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 33de131a28520cc9ecc2bf159dfd6128 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-25T03:39:35,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411256d4279ebae174215bcb178cea4390ce3_33de131a28520cc9ecc2bf159dfd6128 is 50, key is test_row_0/A:col10/1732505975330/Put/seqid=0 2024-11-25T03:39:35,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742364_1540 (size=14794) 2024-11-25T03:39:35,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:35,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39900 deadline: 1732506035423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:35,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:35,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:35,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:35,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:35,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1732506035425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:35,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:35,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:35,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:35,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-25T03:39:35,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-25T03:39:35,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:35,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. as already flushing 2024-11-25T03:39:35,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:35,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:35,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:35,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:35,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:35,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39900 deadline: 1732506035537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:35,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:35,539 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:35,539 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:35,539 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:35,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1732506035537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:35,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:35,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:35,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:35,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-25T03:39:35,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:35,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. as already flushing 2024-11-25T03:39:35,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:35,647 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:35,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:35,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:35,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-25T03:39:35,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:35,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39900 deadline: 1732506035748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:35,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:35,750 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:35,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1732506035748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:35,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:35,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:35,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:35,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:35,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:35,779 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:35,783 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411256d4279ebae174215bcb178cea4390ce3_33de131a28520cc9ecc2bf159dfd6128 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411256d4279ebae174215bcb178cea4390ce3_33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:35,785 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/6bbf8ad142ed43c3b8a20e74c8517dac, store: [table=TestAcidGuarantees family=A region=33de131a28520cc9ecc2bf159dfd6128] 2024-11-25T03:39:35,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/6bbf8ad142ed43c3b8a20e74c8517dac is 175, key is test_row_0/A:col10/1732505975330/Put/seqid=0 2024-11-25T03:39:35,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742365_1541 (size=39749) 2024-11-25T03:39:35,794 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/6bbf8ad142ed43c3b8a20e74c8517dac 2024-11-25T03:39:35,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/5ff203514bb84f55b4e3172dc4fda78a is 50, key is test_row_0/B:col10/1732505975330/Put/seqid=0 2024-11-25T03:39:35,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-25T03:39:35,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:35,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. as already flushing 2024-11-25T03:39:35,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:35,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:35,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:35,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:35,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742366_1542 (size=12151) 2024-11-25T03:39:35,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:35,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1732506035907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:35,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:35,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39884 deadline: 1732506035907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:35,910 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,34881,1732505806588 for 'test_row_0' in 33de131a28520cc9ecc2bf159dfd6128 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:39:35,910 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:35,910 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,34881,1732505806588 for 'test_row_0' in 33de131a28520cc9ecc2bf159dfd6128 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:39:35,910 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:35,910 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:35,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:35,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:35,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:35,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:35,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39858 deadline: 1732506035917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:35,921 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,34881,1732505806588 for 'test_row_2' in 33de131a28520cc9ecc2bf159dfd6128 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:39:35,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:35,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:35,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:35,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-25T03:39:35,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:35,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. as already flushing 2024-11-25T03:39:35,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:35,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:35,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:35,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:35,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-25T03:39:36,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:36,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39900 deadline: 1732506036058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:36,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:36,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:36,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:36,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1732506036058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:36,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:36,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:36,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:36,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:36,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-25T03:39:36,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:36,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. as already flushing 2024-11-25T03:39:36,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:36,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:36,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:36,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:36,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/5ff203514bb84f55b4e3172dc4fda78a 2024-11-25T03:39:36,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/9d0861a43b1c47a8ba7018316825fe3c is 50, key is test_row_0/C:col10/1732505975330/Put/seqid=0 2024-11-25T03:39:36,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742367_1543 (size=12151) 2024-11-25T03:39:36,225 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/9d0861a43b1c47a8ba7018316825fe3c 2024-11-25T03:39:36,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/6bbf8ad142ed43c3b8a20e74c8517dac as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/6bbf8ad142ed43c3b8a20e74c8517dac 2024-11-25T03:39:36,232 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/6bbf8ad142ed43c3b8a20e74c8517dac, entries=200, sequenceid=172, filesize=38.8 K 2024-11-25T03:39:36,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/5ff203514bb84f55b4e3172dc4fda78a as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/5ff203514bb84f55b4e3172dc4fda78a 2024-11-25T03:39:36,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/5ff203514bb84f55b4e3172dc4fda78a, entries=150, sequenceid=172, filesize=11.9 K 2024-11-25T03:39:36,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/9d0861a43b1c47a8ba7018316825fe3c as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/9d0861a43b1c47a8ba7018316825fe3c 2024-11-25T03:39:36,241 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/9d0861a43b1c47a8ba7018316825fe3c, entries=150, sequenceid=172, filesize=11.9 K 2024-11-25T03:39:36,242 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 33de131a28520cc9ecc2bf159dfd6128 in 874ms, sequenceid=172, compaction requested=true 2024-11-25T03:39:36,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 33de131a28520cc9ecc2bf159dfd6128: 2024-11-25T03:39:36,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33de131a28520cc9ecc2bf159dfd6128:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:39:36,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:39:36,242 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:39:36,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33de131a28520cc9ecc2bf159dfd6128:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:39:36,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:39:36,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33de131a28520cc9ecc2bf159dfd6128:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:39:36,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:39:36,242 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:39:36,243 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48644 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:39:36,243 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141698 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:39:36,243 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 33de131a28520cc9ecc2bf159dfd6128/B is initiating minor compaction (all files) 2024-11-25T03:39:36,243 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): 33de131a28520cc9ecc2bf159dfd6128/A is initiating minor compaction (all files) 2024-11-25T03:39:36,243 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33de131a28520cc9ecc2bf159dfd6128/B in TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:36,243 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33de131a28520cc9ecc2bf159dfd6128/A in TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:36,243 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/2835373f0ea3427b9477659423dfbdc5, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/6f831653fa694f90b2e469d85fee9dc4, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/26724993e6b9468d938aac8e6bbe5f56, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/5ff203514bb84f55b4e3172dc4fda78a] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp, totalSize=47.5 K 2024-11-25T03:39:36,243 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/52100752fcf04c22bc0cad7795bbefe8, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/15e329e39f2b4f41bb7befe87e6e17b6, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/98b96364bccd453197ee21c6a15b980f, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/6bbf8ad142ed43c3b8a20e74c8517dac] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp, totalSize=138.4 K 2024-11-25T03:39:36,243 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:36,243 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. files: [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/52100752fcf04c22bc0cad7795bbefe8, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/15e329e39f2b4f41bb7befe87e6e17b6, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/98b96364bccd453197ee21c6a15b980f, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/6bbf8ad142ed43c3b8a20e74c8517dac] 2024-11-25T03:39:36,243 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 2835373f0ea3427b9477659423dfbdc5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732505970576 2024-11-25T03:39:36,244 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52100752fcf04c22bc0cad7795bbefe8, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732505970576 2024-11-25T03:39:36,244 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f831653fa694f90b2e469d85fee9dc4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732505971721 2024-11-25T03:39:36,245 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15e329e39f2b4f41bb7befe87e6e17b6, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732505971721 2024-11-25T03:39:36,245 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 26724993e6b9468d938aac8e6bbe5f56, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732505973143 2024-11-25T03:39:36,245 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98b96364bccd453197ee21c6a15b980f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732505973143 2024-11-25T03:39:36,245 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ff203514bb84f55b4e3172dc4fda78a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732505975330 2024-11-25T03:39:36,245 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6bbf8ad142ed43c3b8a20e74c8517dac, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732505975330 2024-11-25T03:39:36,254 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=33de131a28520cc9ecc2bf159dfd6128] 2024-11-25T03:39:36,255 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33de131a28520cc9ecc2bf159dfd6128#B#compaction#458 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:39:36,255 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/59248e7703bc44268d4d3218765d0235 is 50, key is test_row_0/B:col10/1732505975330/Put/seqid=0 2024-11-25T03:39:36,261 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411256ac492c90060499399de827242836399_33de131a28520cc9ecc2bf159dfd6128 store=[table=TestAcidGuarantees family=A region=33de131a28520cc9ecc2bf159dfd6128] 2024-11-25T03:39:36,263 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411256ac492c90060499399de827242836399_33de131a28520cc9ecc2bf159dfd6128, store=[table=TestAcidGuarantees family=A region=33de131a28520cc9ecc2bf159dfd6128] 2024-11-25T03:39:36,263 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411256ac492c90060499399de827242836399_33de131a28520cc9ecc2bf159dfd6128 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=33de131a28520cc9ecc2bf159dfd6128] 2024-11-25T03:39:36,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-25T03:39:36,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:36,266 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 33de131a28520cc9ecc2bf159dfd6128 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-25T03:39:36,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742368_1544 (size=12527) 2024-11-25T03:39:36,283 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/59248e7703bc44268d4d3218765d0235 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/59248e7703bc44268d4d3218765d0235 2024-11-25T03:39:36,287 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 33de131a28520cc9ecc2bf159dfd6128/B of 33de131a28520cc9ecc2bf159dfd6128 into 59248e7703bc44268d4d3218765d0235(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:39:36,287 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33de131a28520cc9ecc2bf159dfd6128: 2024-11-25T03:39:36,288 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., storeName=33de131a28520cc9ecc2bf159dfd6128/B, priority=12, startTime=1732505976242; duration=0sec 2024-11-25T03:39:36,288 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:39:36,288 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33de131a28520cc9ecc2bf159dfd6128:B 2024-11-25T03:39:36,288 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:39:36,288 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48644 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:39:36,289 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 33de131a28520cc9ecc2bf159dfd6128/C is initiating minor compaction (all files) 2024-11-25T03:39:36,289 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33de131a28520cc9ecc2bf159dfd6128/C in TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:36,289 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/38deacbe2872454ba52d56e687e8e3f3, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/593f2d6d4452449ab090f748a93b917a, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/4fcae415bc81404d823b80aa7f0cd78a, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/9d0861a43b1c47a8ba7018316825fe3c] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp, totalSize=47.5 K 2024-11-25T03:39:36,289 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 38deacbe2872454ba52d56e687e8e3f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732505970576 2024-11-25T03:39:36,289 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 593f2d6d4452449ab090f748a93b917a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732505971721 2024-11-25T03:39:36,290 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fcae415bc81404d823b80aa7f0cd78a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732505973143 2024-11-25T03:39:36,290 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d0861a43b1c47a8ba7018316825fe3c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732505975330 2024-11-25T03:39:36,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742369_1545 (size=4469) 2024-11-25T03:39:36,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125d16cf89c59f94733a7ce9be54a8905ce_33de131a28520cc9ecc2bf159dfd6128 is 50, key is test_row_0/A:col10/1732505975417/Put/seqid=0 2024-11-25T03:39:36,320 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33de131a28520cc9ecc2bf159dfd6128#C#compaction#461 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:39:36,320 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/cd10c323b7364cc68f9e3c8333ef2d1e is 50, key is test_row_0/C:col10/1732505975330/Put/seqid=0 2024-11-25T03:39:36,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742370_1546 (size=12304) 2024-11-25T03:39:36,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:36,335 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125d16cf89c59f94733a7ce9be54a8905ce_33de131a28520cc9ecc2bf159dfd6128 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125d16cf89c59f94733a7ce9be54a8905ce_33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:36,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/792badc607b844d4901bbfe457eee763, store: [table=TestAcidGuarantees family=A region=33de131a28520cc9ecc2bf159dfd6128] 2024-11-25T03:39:36,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/792badc607b844d4901bbfe457eee763 is 175, key is test_row_0/A:col10/1732505975417/Put/seqid=0 2024-11-25T03:39:36,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742371_1547 (size=12527) 2024-11-25T03:39:36,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742372_1548 (size=31105) 2024-11-25T03:39:36,390 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=197, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/792badc607b844d4901bbfe457eee763 2024-11-25T03:39:36,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/52a0bc1b3f374d8fb5f1be373f6a5025 is 50, key is test_row_0/B:col10/1732505975417/Put/seqid=0 2024-11-25T03:39:36,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742373_1549 (size=12151) 2024-11-25T03:39:36,410 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/52a0bc1b3f374d8fb5f1be373f6a5025 2024-11-25T03:39:36,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/3698fd24b35d461ab1fdc9431f1fe424 is 50, key is test_row_0/C:col10/1732505975417/Put/seqid=0 2024-11-25T03:39:36,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742374_1550 (size=12151) 2024-11-25T03:39:36,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-25T03:39:36,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:36,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. as already flushing 2024-11-25T03:39:36,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:36,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39900 deadline: 1732506036589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:36,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:36,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1732506036590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:36,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:36,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:36,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:36,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:36,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:36,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:36,709 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33de131a28520cc9ecc2bf159dfd6128#A#compaction#459 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:39:36,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:36,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39900 deadline: 1732506036707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:36,710 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/d779162afa5e4da99c27f41d37b50207 is 175, key is test_row_0/A:col10/1732505975330/Put/seqid=0 2024-11-25T03:39:36,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:36,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1732506036708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:36,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:36,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:36,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:36,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:36,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:36,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:36,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742375_1551 (size=31481) 2024-11-25T03:39:36,779 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/cd10c323b7364cc68f9e3c8333ef2d1e as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/cd10c323b7364cc68f9e3c8333ef2d1e 2024-11-25T03:39:36,784 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 33de131a28520cc9ecc2bf159dfd6128/C of 33de131a28520cc9ecc2bf159dfd6128 into cd10c323b7364cc68f9e3c8333ef2d1e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:39:36,785 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33de131a28520cc9ecc2bf159dfd6128: 2024-11-25T03:39:36,785 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., storeName=33de131a28520cc9ecc2bf159dfd6128/C, priority=12, startTime=1732505976242; duration=0sec 2024-11-25T03:39:36,785 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:39:36,785 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33de131a28520cc9ecc2bf159dfd6128:C 2024-11-25T03:39:36,827 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/3698fd24b35d461ab1fdc9431f1fe424 2024-11-25T03:39:36,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/792badc607b844d4901bbfe457eee763 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/792badc607b844d4901bbfe457eee763 2024-11-25T03:39:36,833 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/792badc607b844d4901bbfe457eee763, entries=150, sequenceid=197, filesize=30.4 K 2024-11-25T03:39:36,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/52a0bc1b3f374d8fb5f1be373f6a5025 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/52a0bc1b3f374d8fb5f1be373f6a5025 2024-11-25T03:39:36,838 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/52a0bc1b3f374d8fb5f1be373f6a5025, entries=150, sequenceid=197, filesize=11.9 K 2024-11-25T03:39:36,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/3698fd24b35d461ab1fdc9431f1fe424 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/3698fd24b35d461ab1fdc9431f1fe424 2024-11-25T03:39:36,842 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/3698fd24b35d461ab1fdc9431f1fe424, entries=150, sequenceid=197, filesize=11.9 K 2024-11-25T03:39:36,843 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 33de131a28520cc9ecc2bf159dfd6128 in 578ms, sequenceid=197, compaction requested=false 2024-11-25T03:39:36,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 33de131a28520cc9ecc2bf159dfd6128: 2024-11-25T03:39:36,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:36,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-25T03:39:36,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-25T03:39:36,846 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-25T03:39:36,846 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5030 sec 2024-11-25T03:39:36,848 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.5090 sec 2024-11-25T03:39:36,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:36,920 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:39:36,920 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 33de131a28520cc9ecc2bf159dfd6128 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-25T03:39:36,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411253adf353dd3bd47a490b701b98351541c_33de131a28520cc9ecc2bf159dfd6128 is 50, key is test_row_0/A:col10/1732505976919/Put/seqid=0 2024-11-25T03:39:36,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742376_1552 (size=14794) 2024-11-25T03:39:36,944 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:36,948 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411253adf353dd3bd47a490b701b98351541c_33de131a28520cc9ecc2bf159dfd6128 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411253adf353dd3bd47a490b701b98351541c_33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:36,948 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/cec7a41432974390a4976d3c80d3331f, store: [table=TestAcidGuarantees family=A region=33de131a28520cc9ecc2bf159dfd6128] 2024-11-25T03:39:36,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/cec7a41432974390a4976d3c80d3331f is 175, key is test_row_0/A:col10/1732505976919/Put/seqid=0 2024-11-25T03:39:36,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742377_1553 (size=39749) 2024-11-25T03:39:36,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:36,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39900 deadline: 1732506036981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:36,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:36,987 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:36,987 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:36,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1732506036982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:36,987 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:36,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:36,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:36,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:37,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:37,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39900 deadline: 1732506037098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:37,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:37,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1732506037098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:37,101 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:37,101 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:37,101 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:37,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:37,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:37,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:37,118 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/d779162afa5e4da99c27f41d37b50207 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/d779162afa5e4da99c27f41d37b50207 2024-11-25T03:39:37,122 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 33de131a28520cc9ecc2bf159dfd6128/A of 33de131a28520cc9ecc2bf159dfd6128 into d779162afa5e4da99c27f41d37b50207(size=30.7 K), total size for store is 61.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:39:37,122 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33de131a28520cc9ecc2bf159dfd6128: 2024-11-25T03:39:37,122 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., storeName=33de131a28520cc9ecc2bf159dfd6128/A, priority=12, startTime=1732505976242; duration=0sec 2024-11-25T03:39:37,122 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:39:37,122 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33de131a28520cc9ecc2bf159dfd6128:A 2024-11-25T03:39:37,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:37,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39900 deadline: 1732506037307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:37,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:37,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1732506037307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:37,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:37,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:37,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:37,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:37,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:37,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:37,353 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=212, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/cec7a41432974390a4976d3c80d3331f 2024-11-25T03:39:37,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/79b9057170a94f8192c650f7d2af237f is 50, key is test_row_0/B:col10/1732505976919/Put/seqid=0 2024-11-25T03:39:37,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742378_1554 (size=12151) 2024-11-25T03:39:37,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-25T03:39:37,488 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:39:37,488 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:39:37,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-25T03:39:37,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-25T03:39:37,490 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:39:37,491 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:39:37,491 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:39:37,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-25T03:39:37,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:37,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39900 deadline: 1732506037617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:37,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:37,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:37,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:37,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:37,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1732506037617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:37,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:37,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:37,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:37,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-25T03:39:37,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:37,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. as already flushing 2024-11-25T03:39:37,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:37,643 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:37,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:37,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:37,763 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/79b9057170a94f8192c650f7d2af237f 2024-11-25T03:39:37,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/19afaf52d82d45eea862cd5dbaf727a0 is 50, key is test_row_0/C:col10/1732505976919/Put/seqid=0 2024-11-25T03:39:37,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742379_1555 (size=12151) 2024-11-25T03:39:37,772 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/19afaf52d82d45eea862cd5dbaf727a0 2024-11-25T03:39:37,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/cec7a41432974390a4976d3c80d3331f as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/cec7a41432974390a4976d3c80d3331f 2024-11-25T03:39:37,779 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/cec7a41432974390a4976d3c80d3331f, entries=200, sequenceid=212, filesize=38.8 K 2024-11-25T03:39:37,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/79b9057170a94f8192c650f7d2af237f as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/79b9057170a94f8192c650f7d2af237f 2024-11-25T03:39:37,782 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/79b9057170a94f8192c650f7d2af237f, entries=150, sequenceid=212, filesize=11.9 K 2024-11-25T03:39:37,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/19afaf52d82d45eea862cd5dbaf727a0 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/19afaf52d82d45eea862cd5dbaf727a0 2024-11-25T03:39:37,786 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/19afaf52d82d45eea862cd5dbaf727a0, entries=150, sequenceid=212, filesize=11.9 K 2024-11-25T03:39:37,786 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-25T03:39:37,787 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 33de131a28520cc9ecc2bf159dfd6128 in 867ms, sequenceid=212, compaction requested=true 2024-11-25T03:39:37,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 33de131a28520cc9ecc2bf159dfd6128: 2024-11-25T03:39:37,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33de131a28520cc9ecc2bf159dfd6128:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:39:37,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:39:37,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33de131a28520cc9ecc2bf159dfd6128:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:39:37,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:39:37,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33de131a28520cc9ecc2bf159dfd6128:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:39:37,788 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:39:37,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:39:37,788 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:39:37,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-25T03:39:37,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:37,796 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 33de131a28520cc9ecc2bf159dfd6128 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-25T03:39:37,801 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102335 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:39:37,801 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:39:37,801 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): 33de131a28520cc9ecc2bf159dfd6128/A is initiating minor compaction (all files) 2024-11-25T03:39:37,801 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 33de131a28520cc9ecc2bf159dfd6128/B is initiating minor compaction (all files) 2024-11-25T03:39:37,801 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33de131a28520cc9ecc2bf159dfd6128/A in TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:37,801 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33de131a28520cc9ecc2bf159dfd6128/B in TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:37,801 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/d779162afa5e4da99c27f41d37b50207, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/792badc607b844d4901bbfe457eee763, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/cec7a41432974390a4976d3c80d3331f] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp, totalSize=99.9 K 2024-11-25T03:39:37,801 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/59248e7703bc44268d4d3218765d0235, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/52a0bc1b3f374d8fb5f1be373f6a5025, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/79b9057170a94f8192c650f7d2af237f] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp, totalSize=36.0 K 2024-11-25T03:39:37,801 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:37,801 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. files: [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/d779162afa5e4da99c27f41d37b50207, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/792badc607b844d4901bbfe457eee763, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/cec7a41432974390a4976d3c80d3331f] 2024-11-25T03:39:37,802 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 59248e7703bc44268d4d3218765d0235, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732505975330 2024-11-25T03:39:37,802 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting d779162afa5e4da99c27f41d37b50207, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732505975330 2024-11-25T03:39:37,802 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 52a0bc1b3f374d8fb5f1be373f6a5025, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732505975417 2024-11-25T03:39:37,802 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 792badc607b844d4901bbfe457eee763, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732505975417 2024-11-25T03:39:37,803 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 79b9057170a94f8192c650f7d2af237f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732505976575 2024-11-25T03:39:37,803 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting cec7a41432974390a4976d3c80d3331f, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732505976575 2024-11-25T03:39:37,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125acfb2def46de45de9d57e83fe0c2e1e8_33de131a28520cc9ecc2bf159dfd6128 is 50, key is test_row_0/A:col10/1732505976980/Put/seqid=0 2024-11-25T03:39:37,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-25T03:39:37,813 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=33de131a28520cc9ecc2bf159dfd6128] 2024-11-25T03:39:37,814 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33de131a28520cc9ecc2bf159dfd6128#B#compaction#468 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:39:37,814 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/6b07051d69f4480f9d601447feed3ca4 is 50, key is test_row_0/B:col10/1732505976919/Put/seqid=0 2024-11-25T03:39:37,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742380_1556 (size=12304) 2024-11-25T03:39:37,825 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112595e7e241051e4eb888a73356b787461e_33de131a28520cc9ecc2bf159dfd6128 store=[table=TestAcidGuarantees family=A region=33de131a28520cc9ecc2bf159dfd6128] 2024-11-25T03:39:37,827 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112595e7e241051e4eb888a73356b787461e_33de131a28520cc9ecc2bf159dfd6128, store=[table=TestAcidGuarantees family=A region=33de131a28520cc9ecc2bf159dfd6128] 2024-11-25T03:39:37,827 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112595e7e241051e4eb888a73356b787461e_33de131a28520cc9ecc2bf159dfd6128 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=33de131a28520cc9ecc2bf159dfd6128] 2024-11-25T03:39:37,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742381_1557 (size=12629) 2024-11-25T03:39:37,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742382_1558 (size=4469) 2024-11-25T03:39:37,832 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33de131a28520cc9ecc2bf159dfd6128#A#compaction#469 average throughput is 1.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:39:37,833 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/d87d1802b56943acae57e81538fdf741 is 175, key is test_row_0/A:col10/1732505976919/Put/seqid=0 2024-11-25T03:39:37,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742383_1559 (size=31583) 2024-11-25T03:39:38,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-25T03:39:38,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. as already flushing 2024-11-25T03:39:38,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:38,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:38,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39900 deadline: 1732506038152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:38,156 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:38,156 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:38,156 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:38,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:38,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1732506038154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:38,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:38,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:38,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:38,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:38,219 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125acfb2def46de45de9d57e83fe0c2e1e8_33de131a28520cc9ecc2bf159dfd6128 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125acfb2def46de45de9d57e83fe0c2e1e8_33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:38,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/f9658e0b513640c98b7905a692baf416, store: [table=TestAcidGuarantees family=A region=33de131a28520cc9ecc2bf159dfd6128] 2024-11-25T03:39:38,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/f9658e0b513640c98b7905a692baf416 is 175, key is test_row_0/A:col10/1732505976980/Put/seqid=0 2024-11-25T03:39:38,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742384_1560 (size=31105) 2024-11-25T03:39:38,234 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/6b07051d69f4480f9d601447feed3ca4 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/6b07051d69f4480f9d601447feed3ca4 2024-11-25T03:39:38,238 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 33de131a28520cc9ecc2bf159dfd6128/B of 33de131a28520cc9ecc2bf159dfd6128 into 6b07051d69f4480f9d601447feed3ca4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:39:38,238 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33de131a28520cc9ecc2bf159dfd6128: 2024-11-25T03:39:38,239 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., storeName=33de131a28520cc9ecc2bf159dfd6128/B, priority=13, startTime=1732505977788; duration=0sec 2024-11-25T03:39:38,239 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:39:38,239 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33de131a28520cc9ecc2bf159dfd6128:B 2024-11-25T03:39:38,239 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T03:39:38,241 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T03:39:38,241 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 33de131a28520cc9ecc2bf159dfd6128/C is initiating minor compaction (all files) 2024-11-25T03:39:38,241 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33de131a28520cc9ecc2bf159dfd6128/C in TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:38,241 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/cd10c323b7364cc68f9e3c8333ef2d1e, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/3698fd24b35d461ab1fdc9431f1fe424, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/19afaf52d82d45eea862cd5dbaf727a0] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp, totalSize=36.0 K 2024-11-25T03:39:38,241 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting cd10c323b7364cc68f9e3c8333ef2d1e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732505975330 2024-11-25T03:39:38,241 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/d87d1802b56943acae57e81538fdf741 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/d87d1802b56943acae57e81538fdf741 2024-11-25T03:39:38,242 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 3698fd24b35d461ab1fdc9431f1fe424, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732505975417 2024-11-25T03:39:38,243 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 19afaf52d82d45eea862cd5dbaf727a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732505976575 2024-11-25T03:39:38,246 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 33de131a28520cc9ecc2bf159dfd6128/A of 33de131a28520cc9ecc2bf159dfd6128 into d87d1802b56943acae57e81538fdf741(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:39:38,246 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33de131a28520cc9ecc2bf159dfd6128: 2024-11-25T03:39:38,246 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., storeName=33de131a28520cc9ecc2bf159dfd6128/A, priority=13, startTime=1732505977788; duration=0sec 2024-11-25T03:39:38,246 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:39:38,246 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33de131a28520cc9ecc2bf159dfd6128:A 2024-11-25T03:39:38,248 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33de131a28520cc9ecc2bf159dfd6128#C#compaction#470 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:39:38,249 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/7797950a3c944ba2884987fff15fb5f3 is 50, key is test_row_0/C:col10/1732505976919/Put/seqid=0 2024-11-25T03:39:38,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742385_1561 (size=12629) 2024-11-25T03:39:38,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:38,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39900 deadline: 1732506038257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:38,261 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:38,261 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:38,261 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:38,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:38,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1732506038267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:38,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:38,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:38,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:38,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:38,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39900 deadline: 1732506038467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:38,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:38,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:38,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:38,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:38,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1732506038477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:38,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:38,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:38,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:38,624 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=236, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/f9658e0b513640c98b7905a692baf416 2024-11-25T03:39:38,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-25T03:39:38,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/463e870103f142389c146c3e8c1367fa is 50, key is test_row_0/B:col10/1732505976980/Put/seqid=0 2024-11-25T03:39:38,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742386_1562 (size=12151) 2024-11-25T03:39:38,656 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/7797950a3c944ba2884987fff15fb5f3 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/7797950a3c944ba2884987fff15fb5f3 2024-11-25T03:39:38,659 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 33de131a28520cc9ecc2bf159dfd6128/C of 33de131a28520cc9ecc2bf159dfd6128 into 7797950a3c944ba2884987fff15fb5f3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:39:38,659 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33de131a28520cc9ecc2bf159dfd6128: 2024-11-25T03:39:38,660 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., storeName=33de131a28520cc9ecc2bf159dfd6128/C, priority=13, startTime=1732505977788; duration=0sec 2024-11-25T03:39:38,660 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:39:38,660 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33de131a28520cc9ecc2bf159dfd6128:C 2024-11-25T03:39:38,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:38,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39900 deadline: 1732506038778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:38,779 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:38,779 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:38,779 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:38,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:38,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1732506038787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:38,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:38,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:38,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:39,035 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/463e870103f142389c146c3e8c1367fa 2024-11-25T03:39:39,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/993d81635c3d4b2e88099e8ed6745d5c is 50, key is test_row_0/C:col10/1732505976980/Put/seqid=0 2024-11-25T03:39:39,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742387_1563 (size=12151) 2024-11-25T03:39:39,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:39,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39900 deadline: 1732506039287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:39,290 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:39,290 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:39,290 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:39,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:39,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1732506039297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:39,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:39,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:39,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:39,447 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/993d81635c3d4b2e88099e8ed6745d5c 2024-11-25T03:39:39,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/f9658e0b513640c98b7905a692baf416 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/f9658e0b513640c98b7905a692baf416 2024-11-25T03:39:39,454 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/f9658e0b513640c98b7905a692baf416, entries=150, sequenceid=236, filesize=30.4 K 2024-11-25T03:39:39,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/463e870103f142389c146c3e8c1367fa as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/463e870103f142389c146c3e8c1367fa 2024-11-25T03:39:39,459 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/463e870103f142389c146c3e8c1367fa, entries=150, sequenceid=236, filesize=11.9 K 2024-11-25T03:39:39,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/993d81635c3d4b2e88099e8ed6745d5c as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/993d81635c3d4b2e88099e8ed6745d5c 2024-11-25T03:39:39,463 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/993d81635c3d4b2e88099e8ed6745d5c, entries=150, sequenceid=236, filesize=11.9 K 2024-11-25T03:39:39,464 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 33de131a28520cc9ecc2bf159dfd6128 in 1667ms, sequenceid=236, compaction requested=false 2024-11-25T03:39:39,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 33de131a28520cc9ecc2bf159dfd6128: 2024-11-25T03:39:39,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:39,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-25T03:39:39,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-25T03:39:39,466 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-25T03:39:39,467 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9740 sec 2024-11-25T03:39:39,468 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.9790 sec 2024-11-25T03:39:39,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-25T03:39:39,637 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:39:39,638 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:39:39,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-25T03:39:39,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-25T03:39:39,639 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:39:39,640 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:39:39,640 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:39:39,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-25T03:39:39,792 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-25T03:39:39,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:39,793 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 33de131a28520cc9ecc2bf159dfd6128 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-25T03:39:39,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411256e057fb211f644d190cbacad6144ae7c_33de131a28520cc9ecc2bf159dfd6128 is 50, key is test_row_0/A:col10/1732505978152/Put/seqid=0 2024-11-25T03:39:39,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742388_1564 (size=12304) 2024-11-25T03:39:39,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:39,808 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411256e057fb211f644d190cbacad6144ae7c_33de131a28520cc9ecc2bf159dfd6128 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411256e057fb211f644d190cbacad6144ae7c_33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:39,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/0abfba6efe304615915f1d6bcce7f420, store: [table=TestAcidGuarantees family=A region=33de131a28520cc9ecc2bf159dfd6128] 2024-11-25T03:39:39,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/0abfba6efe304615915f1d6bcce7f420 is 175, key is test_row_0/A:col10/1732505978152/Put/seqid=0 2024-11-25T03:39:39,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742389_1565 (size=31105) 2024-11-25T03:39:39,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:39,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. as already flushing 2024-11-25T03:39:39,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-25T03:39:39,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:39,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1732506039986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:39,991 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:39,991 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:39,991 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:39,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:39,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39884 deadline: 1732506039990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:39,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:39,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:39,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:39,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:39,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39858 deadline: 1732506039991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:39,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:39,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:39,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:40,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1732506040097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:40,101 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:40,101 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:40,101 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:40,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:40,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39884 deadline: 1732506040107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:40,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:40,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:40,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39858 deadline: 1732506040107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:40,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:40,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:40,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:40,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:40,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:40,213 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=252, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/0abfba6efe304615915f1d6bcce7f420 2024-11-25T03:39:40,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/96a3dd182d574258b667a6597a7b79c2 is 50, key is test_row_0/B:col10/1732505978152/Put/seqid=0 2024-11-25T03:39:40,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742390_1566 (size=12151) 2024-11-25T03:39:40,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-25T03:39:40,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:40,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39900 deadline: 1732506040307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:40,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:40,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1732506040308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:40,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:40,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1732506040308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:40,313 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:40,313 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:40,313 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:40,313 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,34881,1732505806588 for 'test_row_0' in 33de131a28520cc9ecc2bf159dfd6128 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:39:40,313 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,34881,1732505806588 for 'test_row_1' in 33de131a28520cc9ecc2bf159dfd6128 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:39:40,313 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:40,313 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:40,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:40,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:40,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:40,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:40,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:40,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39858 deadline: 1732506040317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:40,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:40,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39884 deadline: 1732506040317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:40,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:40,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:40,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:40,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:40,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:40,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:40,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:40,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1732506040617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:40,621 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:40,621 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:40,621 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:40,624 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/96a3dd182d574258b667a6597a7b79c2 2024-11-25T03:39:40,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:40,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39858 deadline: 1732506040628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:40,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:40,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:40,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:40,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:40,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39884 deadline: 1732506040628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:40,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:40,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:40,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:40,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/33e18398bcbe40f8a03a7e2e3b907a7d is 50, key is test_row_0/C:col10/1732505978152/Put/seqid=0 2024-11-25T03:39:40,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742391_1567 (size=12151) 2024-11-25T03:39:40,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-25T03:39:41,035 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/33e18398bcbe40f8a03a7e2e3b907a7d 2024-11-25T03:39:41,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/0abfba6efe304615915f1d6bcce7f420 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/0abfba6efe304615915f1d6bcce7f420 2024-11-25T03:39:41,043 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/0abfba6efe304615915f1d6bcce7f420, entries=150, sequenceid=252, filesize=30.4 K 2024-11-25T03:39:41,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/96a3dd182d574258b667a6597a7b79c2 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/96a3dd182d574258b667a6597a7b79c2 2024-11-25T03:39:41,046 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/96a3dd182d574258b667a6597a7b79c2, entries=150, sequenceid=252, filesize=11.9 K 2024-11-25T03:39:41,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/33e18398bcbe40f8a03a7e2e3b907a7d as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/33e18398bcbe40f8a03a7e2e3b907a7d 2024-11-25T03:39:41,050 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/33e18398bcbe40f8a03a7e2e3b907a7d, entries=150, sequenceid=252, filesize=11.9 K 2024-11-25T03:39:41,051 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 33de131a28520cc9ecc2bf159dfd6128 in 1258ms, sequenceid=252, compaction requested=true 2024-11-25T03:39:41,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 33de131a28520cc9ecc2bf159dfd6128: 2024-11-25T03:39:41,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:41,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-25T03:39:41,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-25T03:39:41,054 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-25T03:39:41,054 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4120 sec 2024-11-25T03:39:41,056 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.4170 sec 2024-11-25T03:39:41,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:41,131 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:39:41,131 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 33de131a28520cc9ecc2bf159dfd6128 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-25T03:39:41,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125d51f1e0e1066448b8babcc37278a4592_33de131a28520cc9ecc2bf159dfd6128 is 50, key is test_row_0/A:col10/1732505979989/Put/seqid=0 2024-11-25T03:39:41,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742392_1568 (size=12454) 2024-11-25T03:39:41,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:41,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39884 deadline: 1732506041147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:41,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:41,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:41,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:41,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:41,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39858 deadline: 1732506041147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:41,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:41,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:41,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:41,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:41,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1732506041152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:41,157 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:41,157 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:41,157 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:41,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:41,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39884 deadline: 1732506041257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:41,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:41,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:41,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:41,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:41,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39858 deadline: 1732506041257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:41,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:41,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:41,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:41,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:41,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1732506041267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:41,270 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:41,270 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:41,270 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:41,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:41,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39884 deadline: 1732506041468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:41,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:41,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39858 deadline: 1732506041468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:41,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:41,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:41,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:41,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:41,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:41,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:41,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:41,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1732506041477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:41,481 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:41,481 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:41,481 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:41,541 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:41,544 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125d51f1e0e1066448b8babcc37278a4592_33de131a28520cc9ecc2bf159dfd6128 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125d51f1e0e1066448b8babcc37278a4592_33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:41,545 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/02ae6f0eb345452f92bb1aa7aa0b3ad4, store: [table=TestAcidGuarantees family=A region=33de131a28520cc9ecc2bf159dfd6128] 2024-11-25T03:39:41,545 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/02ae6f0eb345452f92bb1aa7aa0b3ad4 is 175, key is test_row_0/A:col10/1732505979989/Put/seqid=0 2024-11-25T03:39:41,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742393_1569 (size=31255) 2024-11-25T03:39:41,554 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=277, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/02ae6f0eb345452f92bb1aa7aa0b3ad4 2024-11-25T03:39:41,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/d2f61736d23147a28aa31993ae67c20e is 50, key is test_row_0/B:col10/1732505979989/Put/seqid=0 2024-11-25T03:39:41,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742394_1570 (size=12301) 2024-11-25T03:39:41,563 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/d2f61736d23147a28aa31993ae67c20e 2024-11-25T03:39:41,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/d00fcd9ff0fb40a9ae8c284df19a6e5b is 50, key is test_row_0/C:col10/1732505979989/Put/seqid=0 2024-11-25T03:39:41,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742395_1571 (size=12301) 2024-11-25T03:39:41,572 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/d00fcd9ff0fb40a9ae8c284df19a6e5b 2024-11-25T03:39:41,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/02ae6f0eb345452f92bb1aa7aa0b3ad4 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/02ae6f0eb345452f92bb1aa7aa0b3ad4 2024-11-25T03:39:41,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/02ae6f0eb345452f92bb1aa7aa0b3ad4, entries=150, sequenceid=277, filesize=30.5 K 2024-11-25T03:39:41,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/d2f61736d23147a28aa31993ae67c20e as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/d2f61736d23147a28aa31993ae67c20e 2024-11-25T03:39:41,583 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/d2f61736d23147a28aa31993ae67c20e, entries=150, sequenceid=277, filesize=12.0 K 2024-11-25T03:39:41,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/d00fcd9ff0fb40a9ae8c284df19a6e5b as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/d00fcd9ff0fb40a9ae8c284df19a6e5b 2024-11-25T03:39:41,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/d00fcd9ff0fb40a9ae8c284df19a6e5b, entries=150, sequenceid=277, filesize=12.0 K 2024-11-25T03:39:41,587 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 33de131a28520cc9ecc2bf159dfd6128 in 456ms, sequenceid=277, compaction requested=true 2024-11-25T03:39:41,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 33de131a28520cc9ecc2bf159dfd6128: 2024-11-25T03:39:41,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33de131a28520cc9ecc2bf159dfd6128:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:39:41,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:39:41,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33de131a28520cc9ecc2bf159dfd6128:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:39:41,588 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:39:41,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:39:41,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33de131a28520cc9ecc2bf159dfd6128:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:39:41,588 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:39:41,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:39:41,589 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 125048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:39:41,589 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): 33de131a28520cc9ecc2bf159dfd6128/A is initiating minor compaction (all files) 2024-11-25T03:39:41,589 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33de131a28520cc9ecc2bf159dfd6128/A in TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:41,589 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/d87d1802b56943acae57e81538fdf741, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/f9658e0b513640c98b7905a692baf416, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/0abfba6efe304615915f1d6bcce7f420, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/02ae6f0eb345452f92bb1aa7aa0b3ad4] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp, totalSize=122.1 K 2024-11-25T03:39:41,589 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:41,589 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. files: [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/d87d1802b56943acae57e81538fdf741, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/f9658e0b513640c98b7905a692baf416, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/0abfba6efe304615915f1d6bcce7f420, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/02ae6f0eb345452f92bb1aa7aa0b3ad4] 2024-11-25T03:39:41,590 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:39:41,590 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting d87d1802b56943acae57e81538fdf741, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732505976575 2024-11-25T03:39:41,590 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 33de131a28520cc9ecc2bf159dfd6128/B is initiating minor compaction (all files) 2024-11-25T03:39:41,590 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33de131a28520cc9ecc2bf159dfd6128/B in TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:41,590 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9658e0b513640c98b7905a692baf416, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732505976961 2024-11-25T03:39:41,590 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/6b07051d69f4480f9d601447feed3ca4, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/463e870103f142389c146c3e8c1367fa, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/96a3dd182d574258b667a6597a7b79c2, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/d2f61736d23147a28aa31993ae67c20e] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp, totalSize=48.1 K 2024-11-25T03:39:41,590 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0abfba6efe304615915f1d6bcce7f420, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732505978146 2024-11-25T03:39:41,590 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b07051d69f4480f9d601447feed3ca4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732505976575 2024-11-25T03:39:41,590 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02ae6f0eb345452f92bb1aa7aa0b3ad4, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732505979976 2024-11-25T03:39:41,591 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 463e870103f142389c146c3e8c1367fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732505976961 2024-11-25T03:39:41,591 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 96a3dd182d574258b667a6597a7b79c2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732505978146 2024-11-25T03:39:41,592 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting d2f61736d23147a28aa31993ae67c20e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732505979976 2024-11-25T03:39:41,597 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=33de131a28520cc9ecc2bf159dfd6128] 2024-11-25T03:39:41,600 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33de131a28520cc9ecc2bf159dfd6128#B#compaction#480 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:39:41,600 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/675d04b0aa52473a993e7d03de69c7d8 is 50, key is test_row_0/B:col10/1732505979989/Put/seqid=0 2024-11-25T03:39:41,602 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125bf95f4dfad6a41d1a2fbd5b3130819ca_33de131a28520cc9ecc2bf159dfd6128 store=[table=TestAcidGuarantees family=A region=33de131a28520cc9ecc2bf159dfd6128] 2024-11-25T03:39:41,604 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125bf95f4dfad6a41d1a2fbd5b3130819ca_33de131a28520cc9ecc2bf159dfd6128, store=[table=TestAcidGuarantees family=A region=33de131a28520cc9ecc2bf159dfd6128] 2024-11-25T03:39:41,604 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125bf95f4dfad6a41d1a2fbd5b3130819ca_33de131a28520cc9ecc2bf159dfd6128 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=33de131a28520cc9ecc2bf159dfd6128] 2024-11-25T03:39:41,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742396_1572 (size=12915) 2024-11-25T03:39:41,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742397_1573 (size=4469) 2024-11-25T03:39:41,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:41,781 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:39:41,781 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 33de131a28520cc9ecc2bf159dfd6128 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-25T03:39:41,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-25T03:39:41,788 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112521d11eeeff8c4d4aa5a765cebad00866_33de131a28520cc9ecc2bf159dfd6128 is 50, key is test_row_0/A:col10/1732505981781/Put/seqid=0 2024-11-25T03:39:41,788 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:39:41,788 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T03:39:41,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-25T03:39:41,790 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T03:39:41,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-25T03:39:41,791 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T03:39:41,791 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T03:39:41,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742398_1574 (size=14994) 2024-11-25T03:39:41,793 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:41,796 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112521d11eeeff8c4d4aa5a765cebad00866_33de131a28520cc9ecc2bf159dfd6128 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112521d11eeeff8c4d4aa5a765cebad00866_33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:41,797 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/a5376abb18444e42b5f82b02a260294e, store: [table=TestAcidGuarantees family=A region=33de131a28520cc9ecc2bf159dfd6128] 2024-11-25T03:39:41,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/a5376abb18444e42b5f82b02a260294e is 175, key is test_row_0/A:col10/1732505981781/Put/seqid=0 2024-11-25T03:39:41,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742399_1575 (size=39949) 2024-11-25T03:39:41,802 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=290, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/a5376abb18444e42b5f82b02a260294e 2024-11-25T03:39:41,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/bf648520d7924bf191fa7da414b187c7 is 50, key is test_row_0/B:col10/1732505981781/Put/seqid=0 2024-11-25T03:39:41,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742400_1576 (size=12301) 2024-11-25T03:39:41,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:41,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1732506041846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:41,852 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:41,852 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:41,852 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:41,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:41,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39884 deadline: 1732506041850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:41,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:41,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:41,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:41,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39858 deadline: 1732506041851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:41,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:41,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:41,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:41,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:41,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-25T03:39:41,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-25T03:39:41,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:41,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. as already flushing 2024-11-25T03:39:41,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:41,944 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:41,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:41,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:41,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:41,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1732506041957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:41,959 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:41,959 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:41,959 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:41,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:41,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39884 deadline: 1732506041968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:41,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:41,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:41,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:41,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39858 deadline: 1732506041968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:41,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:41,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:41,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:41,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:42,009 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/675d04b0aa52473a993e7d03de69c7d8 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/675d04b0aa52473a993e7d03de69c7d8 2024-11-25T03:39:42,012 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 33de131a28520cc9ecc2bf159dfd6128/B of 33de131a28520cc9ecc2bf159dfd6128 into 675d04b0aa52473a993e7d03de69c7d8(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:39:42,012 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33de131a28520cc9ecc2bf159dfd6128: 2024-11-25T03:39:42,012 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., storeName=33de131a28520cc9ecc2bf159dfd6128/B, priority=12, startTime=1732505981587; duration=0sec 2024-11-25T03:39:42,013 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:39:42,013 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33de131a28520cc9ecc2bf159dfd6128:B 2024-11-25T03:39:42,013 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:39:42,014 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:39:42,014 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 33de131a28520cc9ecc2bf159dfd6128/C is initiating minor compaction (all files) 2024-11-25T03:39:42,014 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33de131a28520cc9ecc2bf159dfd6128/C in TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:42,014 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/7797950a3c944ba2884987fff15fb5f3, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/993d81635c3d4b2e88099e8ed6745d5c, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/33e18398bcbe40f8a03a7e2e3b907a7d, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/d00fcd9ff0fb40a9ae8c284df19a6e5b] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp, totalSize=48.1 K 2024-11-25T03:39:42,014 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33de131a28520cc9ecc2bf159dfd6128#A#compaction#479 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:39:42,014 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 7797950a3c944ba2884987fff15fb5f3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732505976575 2024-11-25T03:39:42,015 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 993d81635c3d4b2e88099e8ed6745d5c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732505976961 2024-11-25T03:39:42,015 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/7523289aec1f4bd7a2b48735b8ab96db is 175, key is test_row_0/A:col10/1732505979989/Put/seqid=0 2024-11-25T03:39:42,015 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 33e18398bcbe40f8a03a7e2e3b907a7d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732505978146 2024-11-25T03:39:42,016 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting d00fcd9ff0fb40a9ae8c284df19a6e5b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732505979976 2024-11-25T03:39:42,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742401_1577 (size=31869) 2024-11-25T03:39:42,022 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/7523289aec1f4bd7a2b48735b8ab96db as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/7523289aec1f4bd7a2b48735b8ab96db 2024-11-25T03:39:42,023 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33de131a28520cc9ecc2bf159dfd6128#C#compaction#483 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:39:42,023 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/b1ee2ed0769f4eab80bb286bcf25d77f is 50, key is test_row_0/C:col10/1732505979989/Put/seqid=0 2024-11-25T03:39:42,026 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 33de131a28520cc9ecc2bf159dfd6128/A of 33de131a28520cc9ecc2bf159dfd6128 into 7523289aec1f4bd7a2b48735b8ab96db(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:39:42,026 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33de131a28520cc9ecc2bf159dfd6128: 2024-11-25T03:39:42,026 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., storeName=33de131a28520cc9ecc2bf159dfd6128/A, priority=12, startTime=1732505981587; duration=0sec 2024-11-25T03:39:42,026 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:39:42,026 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33de131a28520cc9ecc2bf159dfd6128:A 2024-11-25T03:39:42,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742402_1578 (size=12915) 2024-11-25T03:39:42,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-25T03:39:42,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:42,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. as already flushing 2024-11-25T03:39:42,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:42,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:42,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:42,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:42,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-25T03:39:42,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:42,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1732506042167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:42,171 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:42,171 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:42,171 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:42,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:42,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39884 deadline: 1732506042177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:42,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:42,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:42,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39858 deadline: 1732506042178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:42,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:42,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:42,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:42,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:42,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:42,213 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/bf648520d7924bf191fa7da414b187c7 2024-11-25T03:39:42,219 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/3bf041c4ac1f493f9db2e7e2109b93e7 is 50, key is test_row_0/C:col10/1732505981781/Put/seqid=0 2024-11-25T03:39:42,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742403_1579 (size=12301) 2024-11-25T03:39:42,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-25T03:39:42,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:42,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. as already flushing 2024-11-25T03:39:42,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:42,249 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:42,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:42,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:42,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:42,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39900 deadline: 1732506042337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:42,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:42,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1732506042337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:42,340 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,34881,1732505806588 for 'test_row_0' in 33de131a28520cc9ecc2bf159dfd6128 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:39:42,340 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:42,340 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:42,340 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:42,340 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 38a5d2077268,34881,1732505806588 for 'test_row_1' in 33de131a28520cc9ecc2bf159dfd6128 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-25T03:39:42,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:42,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:42,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:42,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-25T03:39:42,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:42,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. as already flushing 2024-11-25T03:39:42,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:42,402 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:42,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:42,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:42,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-25T03:39:42,431 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/b1ee2ed0769f4eab80bb286bcf25d77f as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/b1ee2ed0769f4eab80bb286bcf25d77f 2024-11-25T03:39:42,435 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 33de131a28520cc9ecc2bf159dfd6128/C of 33de131a28520cc9ecc2bf159dfd6128 into b1ee2ed0769f4eab80bb286bcf25d77f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:39:42,435 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33de131a28520cc9ecc2bf159dfd6128: 2024-11-25T03:39:42,435 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., storeName=33de131a28520cc9ecc2bf159dfd6128/C, priority=12, startTime=1732505981588; duration=0sec 2024-11-25T03:39:42,435 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:39:42,435 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33de131a28520cc9ecc2bf159dfd6128:C 2024-11-25T03:39:42,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:42,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1732506042477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:42,478 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:42,478 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:42,478 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:42,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:42,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39884 deadline: 1732506042487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:42,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:42,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39858 deadline: 1732506042487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:42,490 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:42,490 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:42,490 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:42,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:42,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:42,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:42,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-25T03:39:42,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:42,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. as already flushing 2024-11-25T03:39:42,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:42,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:42,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:42,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:42,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/3bf041c4ac1f493f9db2e7e2109b93e7 2024-11-25T03:39:42,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/a5376abb18444e42b5f82b02a260294e as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/a5376abb18444e42b5f82b02a260294e 2024-11-25T03:39:42,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/a5376abb18444e42b5f82b02a260294e, entries=200, sequenceid=290, filesize=39.0 K 2024-11-25T03:39:42,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/bf648520d7924bf191fa7da414b187c7 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/bf648520d7924bf191fa7da414b187c7 2024-11-25T03:39:42,633 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/bf648520d7924bf191fa7da414b187c7, entries=150, sequenceid=290, filesize=12.0 K 2024-11-25T03:39:42,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/3bf041c4ac1f493f9db2e7e2109b93e7 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/3bf041c4ac1f493f9db2e7e2109b93e7 2024-11-25T03:39:42,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/3bf041c4ac1f493f9db2e7e2109b93e7, entries=150, sequenceid=290, filesize=12.0 K 2024-11-25T03:39:42,637 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 33de131a28520cc9ecc2bf159dfd6128 in 856ms, sequenceid=290, compaction requested=false 2024-11-25T03:39:42,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 33de131a28520cc9ecc2bf159dfd6128: 2024-11-25T03:39:42,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34881 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-25T03:39:42,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:42,707 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 33de131a28520cc9ecc2bf159dfd6128 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-25T03:39:42,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411255aa2b0259f0341f794e63d086b70ae87_33de131a28520cc9ecc2bf159dfd6128 is 50, key is test_row_0/A:col10/1732505981850/Put/seqid=0 2024-11-25T03:39:42,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742404_1580 (size=12454) 2024-11-25T03:39:42,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-25T03:39:42,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:42,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. as already flushing 2024-11-25T03:39:43,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:43,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39858 deadline: 1732506043005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:43,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:43,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:43,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:43,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:43,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1732506043008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:43,012 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:43,012 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:43,012 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:43,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:43,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39884 deadline: 1732506043010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:43,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:43,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:43,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:43,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:43,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:43,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39858 deadline: 1732506043117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:43,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:43,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:43,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:43,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1732506043117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:43,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:43,121 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:43,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:43,121 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:43,121 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:43,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39884 deadline: 1732506043117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:43,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:43,121 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411255aa2b0259f0341f794e63d086b70ae87_33de131a28520cc9ecc2bf159dfd6128 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411255aa2b0259f0341f794e63d086b70ae87_33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:43,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:43,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:43,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/c16eb552f40b4bf7a9d9e9b0f323574d, store: [table=TestAcidGuarantees family=A region=33de131a28520cc9ecc2bf159dfd6128] 2024-11-25T03:39:43,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/c16eb552f40b4bf7a9d9e9b0f323574d is 175, key is test_row_0/A:col10/1732505981850/Put/seqid=0 2024-11-25T03:39:43,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742405_1581 (size=31255) 2024-11-25T03:39:43,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:43,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39858 deadline: 1732506043327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:43,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:43,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:43,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1732506043327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:43,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:43,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:43,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:43,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39884 deadline: 1732506043327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:43,331 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:43,331 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:43,331 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:43,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:43,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:43,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:43,526 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=316, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/c16eb552f40b4bf7a9d9e9b0f323574d 2024-11-25T03:39:43,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/e5db130b3a364fceb212e60ac16107c8 is 50, key is test_row_0/B:col10/1732505981850/Put/seqid=0 2024-11-25T03:39:43,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742406_1582 (size=12301) 2024-11-25T03:39:43,542 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/e5db130b3a364fceb212e60ac16107c8 2024-11-25T03:39:43,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/855577da0db044528d6beb50bcc2b622 is 50, key is test_row_0/C:col10/1732505981850/Put/seqid=0 2024-11-25T03:39:43,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742407_1583 (size=12301) 2024-11-25T03:39:43,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:43,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39858 deadline: 1732506043637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:43,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:43,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:43,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39884 deadline: 1732506043638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:43,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:43,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:43,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:43,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1732506043638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:43,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:43,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:43,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:43,641 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:43,641 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:43,641 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:43,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-25T03:39:43,951 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/855577da0db044528d6beb50bcc2b622 2024-11-25T03:39:43,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/c16eb552f40b4bf7a9d9e9b0f323574d as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/c16eb552f40b4bf7a9d9e9b0f323574d 2024-11-25T03:39:43,958 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/c16eb552f40b4bf7a9d9e9b0f323574d, entries=150, sequenceid=316, filesize=30.5 K 2024-11-25T03:39:43,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/e5db130b3a364fceb212e60ac16107c8 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/e5db130b3a364fceb212e60ac16107c8 2024-11-25T03:39:43,962 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/e5db130b3a364fceb212e60ac16107c8, entries=150, sequenceid=316, filesize=12.0 K 2024-11-25T03:39:43,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/855577da0db044528d6beb50bcc2b622 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/855577da0db044528d6beb50bcc2b622 2024-11-25T03:39:43,965 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/855577da0db044528d6beb50bcc2b622, entries=150, sequenceid=316, filesize=12.0 K 2024-11-25T03:39:43,966 INFO [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 33de131a28520cc9ecc2bf159dfd6128 in 1259ms, sequenceid=316, compaction requested=true 2024-11-25T03:39:43,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 33de131a28520cc9ecc2bf159dfd6128: 2024-11-25T03:39:43,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:43,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-25T03:39:43,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-25T03:39:43,969 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-25T03:39:43,969 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1760 sec 2024-11-25T03:39:43,971 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.1820 sec 2024-11-25T03:39:44,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:44,150 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:39:44,150 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 33de131a28520cc9ecc2bf159dfd6128 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-25T03:39:44,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125012476379ece44a8a88571c5e5c066f2_33de131a28520cc9ecc2bf159dfd6128 is 50, key is test_row_0/A:col10/1732505983006/Put/seqid=0 2024-11-25T03:39:44,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742408_1584 (size=14994) 2024-11-25T03:39:44,166 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:44,170 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125012476379ece44a8a88571c5e5c066f2_33de131a28520cc9ecc2bf159dfd6128 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125012476379ece44a8a88571c5e5c066f2_33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:44,171 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/c5e3f8aa97fe4068818cba6b72e484c1, store: [table=TestAcidGuarantees family=A region=33de131a28520cc9ecc2bf159dfd6128] 2024-11-25T03:39:44,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/c5e3f8aa97fe4068818cba6b72e484c1 is 175, key is test_row_0/A:col10/1732505983006/Put/seqid=0 2024-11-25T03:39:44,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742409_1585 (size=39949) 2024-11-25T03:39:44,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:44,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1732506044182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:44,186 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:44,186 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:44,186 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:44,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:44,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39858 deadline: 1732506044183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:44,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:44,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:44,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:44,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39884 deadline: 1732506044184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:44,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:44,188 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:44,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:44,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:44,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:44,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1732506044298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:44,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:44,301 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:44,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39858 deadline: 1732506044298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:44,301 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:44,301 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:44,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:44,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:44,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:44,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:44,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39884 deadline: 1732506044298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:44,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:44,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:44,302 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:44,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:44,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1732506044507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:44,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:44,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39858 deadline: 1732506044508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:44,510 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:44,510 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:44,510 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:44,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:44,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39884 deadline: 1732506044508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:44,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:44,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:44,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:44,511 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:44,511 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:44,511 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:44,575 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=331, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/c5e3f8aa97fe4068818cba6b72e484c1 2024-11-25T03:39:44,581 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/cd76d562e26d42b38346a4151b88a33c is 50, key is test_row_0/B:col10/1732505983006/Put/seqid=0 2024-11-25T03:39:44,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742410_1586 (size=12301) 2024-11-25T03:39:44,799 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T03:39:44,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:44,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1732506044817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:44,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:44,821 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:44,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39858 deadline: 1732506044817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:44,821 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:44,821 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:44,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:44,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:44,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:44,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:44,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39884 deadline: 1732506044817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:44,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:44,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:44,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:44,852 INFO [Thread-2267 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2267. 2024-11-25T03:39:44,852 DEBUG [Thread-2267 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:39:44,852 DEBUG [Thread-2267 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:44,852 DEBUG [Thread-2267 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:44,852 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:39:44,855 INFO [Thread-2261 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2261. 2024-11-25T03:39:44,855 INFO [Thread-2265 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2265. 2024-11-25T03:39:44,855 DEBUG [Thread-2261 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:39:44,855 DEBUG [Thread-2261 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:44,855 DEBUG [Thread-2265 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:39:44,855 DEBUG [Thread-2265 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:44,855 DEBUG [Thread-2265 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:44,855 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:39:44,855 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:39:44,855 DEBUG [Thread-2261 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:44,856 INFO [Thread-2259 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2259. 2024-11-25T03:39:44,856 DEBUG [Thread-2259 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:39:44,856 DEBUG [Thread-2259 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:44,856 DEBUG [Thread-2259 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:44,856 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:39:44,858 INFO [Thread-2263 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2263. 2024-11-25T03:39:44,858 DEBUG [Thread-2263 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:39:44,858 DEBUG [Thread-2263 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:44,858 DEBUG [Thread-2263 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:44,858 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:39:44,985 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/cd76d562e26d42b38346a4151b88a33c 2024-11-25T03:39:44,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/54dfa070984a45dda35e89303264c465 is 50, key is test_row_0/C:col10/1732505983006/Put/seqid=0 2024-11-25T03:39:44,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742411_1587 (size=12301) 2024-11-25T03:39:45,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:45,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:45,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T03:39:45,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39858 deadline: 1732506045328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:45,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1732506045328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:45,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34881 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39884 deadline: 1732506045328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 2024-11-25T03:39:45,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:45,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:45,328 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 , the old value is region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:45,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:45,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:45,328 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:39:45,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:45,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:45,328 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., hostname=38a5d2077268,34881,1732505806588, seqNum=5 because the exception is null or not the one we care about 2024-11-25T03:39:45,393 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/54dfa070984a45dda35e89303264c465 2024-11-25T03:39:45,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/c5e3f8aa97fe4068818cba6b72e484c1 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/c5e3f8aa97fe4068818cba6b72e484c1 2024-11-25T03:39:45,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/c5e3f8aa97fe4068818cba6b72e484c1, entries=200, sequenceid=331, filesize=39.0 K 2024-11-25T03:39:45,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/cd76d562e26d42b38346a4151b88a33c as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/cd76d562e26d42b38346a4151b88a33c 2024-11-25T03:39:45,401 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/cd76d562e26d42b38346a4151b88a33c, entries=150, sequenceid=331, filesize=12.0 K 2024-11-25T03:39:45,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/54dfa070984a45dda35e89303264c465 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/54dfa070984a45dda35e89303264c465 2024-11-25T03:39:45,404 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/54dfa070984a45dda35e89303264c465, entries=150, sequenceid=331, filesize=12.0 K 2024-11-25T03:39:45,405 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 33de131a28520cc9ecc2bf159dfd6128 in 1254ms, sequenceid=331, compaction requested=true 2024-11-25T03:39:45,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 33de131a28520cc9ecc2bf159dfd6128: 2024-11-25T03:39:45,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33de131a28520cc9ecc2bf159dfd6128:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T03:39:45,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:39:45,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33de131a28520cc9ecc2bf159dfd6128:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T03:39:45,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:39:45,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33de131a28520cc9ecc2bf159dfd6128:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T03:39:45,405 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:39:45,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:39:45,405 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:39:45,406 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:39:45,406 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 143022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:39:45,406 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 33de131a28520cc9ecc2bf159dfd6128/B is initiating minor compaction (all files) 2024-11-25T03:39:45,406 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1541): 33de131a28520cc9ecc2bf159dfd6128/A is initiating minor compaction (all files) 2024-11-25T03:39:45,406 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33de131a28520cc9ecc2bf159dfd6128/B in TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:45,406 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33de131a28520cc9ecc2bf159dfd6128/A in TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:45,406 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/675d04b0aa52473a993e7d03de69c7d8, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/bf648520d7924bf191fa7da414b187c7, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/e5db130b3a364fceb212e60ac16107c8, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/cd76d562e26d42b38346a4151b88a33c] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp, totalSize=48.7 K 2024-11-25T03:39:45,406 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/7523289aec1f4bd7a2b48735b8ab96db, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/a5376abb18444e42b5f82b02a260294e, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/c16eb552f40b4bf7a9d9e9b0f323574d, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/c5e3f8aa97fe4068818cba6b72e484c1] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp, totalSize=139.7 K 2024-11-25T03:39:45,406 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:45,406 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. files: [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/7523289aec1f4bd7a2b48735b8ab96db, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/a5376abb18444e42b5f82b02a260294e, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/c16eb552f40b4bf7a9d9e9b0f323574d, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/c5e3f8aa97fe4068818cba6b72e484c1] 2024-11-25T03:39:45,406 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 675d04b0aa52473a993e7d03de69c7d8, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732505979976 2024-11-25T03:39:45,406 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7523289aec1f4bd7a2b48735b8ab96db, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732505979976 2024-11-25T03:39:45,406 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting bf648520d7924bf191fa7da414b187c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732505981150 2024-11-25T03:39:45,406 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5376abb18444e42b5f82b02a260294e, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732505981145 2024-11-25T03:39:45,406 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting e5db130b3a364fceb212e60ac16107c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1732505981809 2024-11-25T03:39:45,407 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting c16eb552f40b4bf7a9d9e9b0f323574d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1732505981809 2024-11-25T03:39:45,407 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting cd76d562e26d42b38346a4151b88a33c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732505983006 2024-11-25T03:39:45,407 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5e3f8aa97fe4068818cba6b72e484c1, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732505983006 2024-11-25T03:39:45,411 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=33de131a28520cc9ecc2bf159dfd6128] 2024-11-25T03:39:45,412 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33de131a28520cc9ecc2bf159dfd6128#B#compaction#491 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:39:45,413 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125c899f1bb5b644074984449c3a31014ca_33de131a28520cc9ecc2bf159dfd6128 store=[table=TestAcidGuarantees family=A region=33de131a28520cc9ecc2bf159dfd6128] 2024-11-25T03:39:45,413 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/dc962066135347938ee9f535575f15cb is 50, key is test_row_0/B:col10/1732505983006/Put/seqid=0 2024-11-25T03:39:45,416 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125c899f1bb5b644074984449c3a31014ca_33de131a28520cc9ecc2bf159dfd6128, store=[table=TestAcidGuarantees family=A region=33de131a28520cc9ecc2bf159dfd6128] 2024-11-25T03:39:45,416 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125c899f1bb5b644074984449c3a31014ca_33de131a28520cc9ecc2bf159dfd6128 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=33de131a28520cc9ecc2bf159dfd6128] 2024-11-25T03:39:45,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742412_1588 (size=13051) 2024-11-25T03:39:45,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742413_1589 (size=4469) 2024-11-25T03:39:45,820 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33de131a28520cc9ecc2bf159dfd6128#A#compaction#492 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:39:45,820 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/fafab48df4e34d68bca8e986b308425a is 175, key is test_row_0/A:col10/1732505983006/Put/seqid=0 2024-11-25T03:39:45,820 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/dc962066135347938ee9f535575f15cb as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/dc962066135347938ee9f535575f15cb 2024-11-25T03:39:45,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742414_1590 (size=32005) 2024-11-25T03:39:45,825 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 33de131a28520cc9ecc2bf159dfd6128/B of 33de131a28520cc9ecc2bf159dfd6128 into dc962066135347938ee9f535575f15cb(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:39:45,825 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33de131a28520cc9ecc2bf159dfd6128: 2024-11-25T03:39:45,825 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., storeName=33de131a28520cc9ecc2bf159dfd6128/B, priority=12, startTime=1732505985405; duration=0sec 2024-11-25T03:39:45,825 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T03:39:45,825 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33de131a28520cc9ecc2bf159dfd6128:B 2024-11-25T03:39:45,825 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:39:45,826 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T03:39:45,826 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1541): 33de131a28520cc9ecc2bf159dfd6128/C is initiating minor compaction (all files) 2024-11-25T03:39:45,826 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33de131a28520cc9ecc2bf159dfd6128/C in TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:45,826 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/b1ee2ed0769f4eab80bb286bcf25d77f, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/3bf041c4ac1f493f9db2e7e2109b93e7, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/855577da0db044528d6beb50bcc2b622, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/54dfa070984a45dda35e89303264c465] into tmpdir=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp, totalSize=48.7 K 2024-11-25T03:39:45,826 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting b1ee2ed0769f4eab80bb286bcf25d77f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732505979976 2024-11-25T03:39:45,826 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bf041c4ac1f493f9db2e7e2109b93e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732505981150 2024-11-25T03:39:45,827 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 855577da0db044528d6beb50bcc2b622, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1732505981809 2024-11-25T03:39:45,827 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] compactions.Compactor(225): Compacting 54dfa070984a45dda35e89303264c465, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732505983006 2024-11-25T03:39:45,827 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/fafab48df4e34d68bca8e986b308425a as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/fafab48df4e34d68bca8e986b308425a 2024-11-25T03:39:45,831 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 33de131a28520cc9ecc2bf159dfd6128/A of 33de131a28520cc9ecc2bf159dfd6128 into fafab48df4e34d68bca8e986b308425a(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:39:45,831 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33de131a28520cc9ecc2bf159dfd6128: 2024-11-25T03:39:45,831 INFO [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., storeName=33de131a28520cc9ecc2bf159dfd6128/A, priority=12, startTime=1732505985405; duration=0sec 2024-11-25T03:39:45,831 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:39:45,831 DEBUG [RS:0;38a5d2077268:34881-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33de131a28520cc9ecc2bf159dfd6128:A 2024-11-25T03:39:45,833 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33de131a28520cc9ecc2bf159dfd6128#C#compaction#493 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:39:45,834 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/7755be0a89b648748eece004c886193d is 50, key is test_row_0/C:col10/1732505983006/Put/seqid=0 2024-11-25T03:39:45,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742415_1591 (size=13051) 2024-11-25T03:39:45,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-25T03:39:45,967 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:39:46,240 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/7755be0a89b648748eece004c886193d as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/7755be0a89b648748eece004c886193d 2024-11-25T03:39:46,243 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 33de131a28520cc9ecc2bf159dfd6128/C of 33de131a28520cc9ecc2bf159dfd6128 into 7755be0a89b648748eece004c886193d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:39:46,243 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33de131a28520cc9ecc2bf159dfd6128: 2024-11-25T03:39:46,243 INFO [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., storeName=33de131a28520cc9ecc2bf159dfd6128/C, priority=12, startTime=1732505985405; duration=0sec 2024-11-25T03:39:46,243 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T03:39:46,243 DEBUG [RS:0;38a5d2077268:34881-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33de131a28520cc9ecc2bf159dfd6128:C 2024-11-25T03:39:46,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34881 {}] regionserver.HRegion(8855): Flush requested on 33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:46,338 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T03:39:46,338 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 33de131a28520cc9ecc2bf159dfd6128 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-25T03:39:46,339 INFO [Thread-2252 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2252. 2024-11-25T03:39:46,339 DEBUG [Thread-2252 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:39:46,339 DEBUG [Thread-2252 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:46,339 DEBUG [Thread-2252 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:46,339 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:39:46,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125b220086c25824c8d8ab02146331abcc5_33de131a28520cc9ecc2bf159dfd6128 is 50, key is test_row_0/A:col10/1732505986337/Put/seqid=0 2024-11-25T03:39:46,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742416_1592 (size=12454) 2024-11-25T03:39:46,349 INFO [Thread-2254 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2254. 2024-11-25T03:39:46,349 INFO [Thread-2248 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2248. 2024-11-25T03:39:46,349 DEBUG [Thread-2254 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:39:46,349 DEBUG [Thread-2254 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:46,349 DEBUG [Thread-2248 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:39:46,349 DEBUG [Thread-2248 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:46,349 DEBUG [Thread-2254 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:46,349 DEBUG [Thread-2248 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:46,349 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:39:46,349 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:39:46,359 INFO [Thread-2256 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2256. 2024-11-25T03:39:46,359 INFO [Thread-2250 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2250. 2024-11-25T03:39:46,359 DEBUG [Thread-2256 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:39:46,359 DEBUG [Thread-2250 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T03:39:46,359 DEBUG [Thread-2256 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:46,359 DEBUG [Thread-2250 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:46,359 DEBUG [Thread-2250 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:46,359 DEBUG [Thread-2256 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:46,359 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:39:46,359 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:39:46,359 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-25T03:39:46,359 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-25T03:39:46,359 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-25T03:39:46,359 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-25T03:39:46,359 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-25T03:39:46,359 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-25T03:39:46,359 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-25T03:39:46,359 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3366 2024-11-25T03:39:46,359 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3382 2024-11-25T03:39:46,359 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3395 2024-11-25T03:39:46,359 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3362 2024-11-25T03:39:46,359 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3358 2024-11-25T03:39:46,359 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-25T03:39:46,359 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T03:39:46,359 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T03:39:46,359 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:46,360 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:46,360 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:39:46,360 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-25T03:39:46,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-25T03:39:46,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-25T03:39:46,363 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732505986363"}]},"ts":"1732505986363"} 2024-11-25T03:39:46,364 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-25T03:39:46,364 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-25T03:39:46,364 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T03:39:46,365 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=33de131a28520cc9ecc2bf159dfd6128, UNASSIGN}] 2024-11-25T03:39:46,366 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=33de131a28520cc9ecc2bf159dfd6128, UNASSIGN 2024-11-25T03:39:46,366 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=33de131a28520cc9ecc2bf159dfd6128, regionState=CLOSING, regionLocation=38a5d2077268,34881,1732505806588 2024-11-25T03:39:46,367 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=33de131a28520cc9ecc2bf159dfd6128, UNASSIGN because future has completed 2024-11-25T03:39:46,367 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T03:39:46,368 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure 33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588}] 2024-11-25T03:39:46,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-25T03:39:46,519 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(122): Close 33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:46,519 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T03:39:46,519 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1722): Closing 33de131a28520cc9ecc2bf159dfd6128, disabling compactions & flushes 2024-11-25T03:39:46,519 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:46,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-25T03:39:46,746 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:46,749 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125b220086c25824c8d8ab02146331abcc5_33de131a28520cc9ecc2bf159dfd6128 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125b220086c25824c8d8ab02146331abcc5_33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:46,750 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/518dbe69c7bc4770ab9c7601a3783373, store: [table=TestAcidGuarantees family=A region=33de131a28520cc9ecc2bf159dfd6128] 2024-11-25T03:39:46,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/518dbe69c7bc4770ab9c7601a3783373 is 175, key is test_row_0/A:col10/1732505986337/Put/seqid=0 2024-11-25T03:39:46,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742417_1593 (size=31255) 2024-11-25T03:39:46,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-25T03:39:47,154 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=356, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/518dbe69c7bc4770ab9c7601a3783373 2024-11-25T03:39:47,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/6cbfee9aef6e47ab95eb0219239d0ac6 is 50, key is test_row_0/B:col10/1732505986337/Put/seqid=0 2024-11-25T03:39:47,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742418_1594 (size=12301) 2024-11-25T03:39:47,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-25T03:39:47,563 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/6cbfee9aef6e47ab95eb0219239d0ac6 2024-11-25T03:39:47,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/150ffee76fae4735826067ee524e44d2 is 50, key is test_row_0/C:col10/1732505986337/Put/seqid=0 2024-11-25T03:39:47,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742419_1595 (size=12301) 2024-11-25T03:39:47,971 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/150ffee76fae4735826067ee524e44d2 2024-11-25T03:39:47,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/518dbe69c7bc4770ab9c7601a3783373 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/518dbe69c7bc4770ab9c7601a3783373 2024-11-25T03:39:47,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/518dbe69c7bc4770ab9c7601a3783373, entries=150, sequenceid=356, filesize=30.5 K 2024-11-25T03:39:47,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/6cbfee9aef6e47ab95eb0219239d0ac6 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/6cbfee9aef6e47ab95eb0219239d0ac6 2024-11-25T03:39:47,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/6cbfee9aef6e47ab95eb0219239d0ac6, entries=150, sequenceid=356, filesize=12.0 K 2024-11-25T03:39:47,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/150ffee76fae4735826067ee524e44d2 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/150ffee76fae4735826067ee524e44d2 2024-11-25T03:39:47,982 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/150ffee76fae4735826067ee524e44d2, entries=150, sequenceid=356, filesize=12.0 K 2024-11-25T03:39:47,983 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=26.84 KB/27480 for 33de131a28520cc9ecc2bf159dfd6128 in 1645ms, sequenceid=356, compaction requested=false 2024-11-25T03:39:47,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 33de131a28520cc9ecc2bf159dfd6128: 2024-11-25T03:39:47,983 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:47,983 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:47,983 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. after waiting 0 ms 2024-11-25T03:39:47,983 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:47,983 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(2902): Flushing 33de131a28520cc9ecc2bf159dfd6128 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-25T03:39:47,987 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125a13bcf0bd9d2414295b9cea669c54580_33de131a28520cc9ecc2bf159dfd6128 is 50, key is test_row_0/A:col10/1732505986358/Put/seqid=0 2024-11-25T03:39:47,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742420_1596 (size=9914) 2024-11-25T03:39:48,390 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:48,393 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125a13bcf0bd9d2414295b9cea669c54580_33de131a28520cc9ecc2bf159dfd6128 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125a13bcf0bd9d2414295b9cea669c54580_33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:48,393 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/ec8a135e774d435281748c179d9ac1a8, store: [table=TestAcidGuarantees family=A region=33de131a28520cc9ecc2bf159dfd6128] 2024-11-25T03:39:48,394 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/ec8a135e774d435281748c179d9ac1a8 is 175, key is test_row_0/A:col10/1732505986358/Put/seqid=0 2024-11-25T03:39:48,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742421_1597 (size=22561) 2024-11-25T03:39:48,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-25T03:39:48,797 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=363, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/ec8a135e774d435281748c179d9ac1a8 2024-11-25T03:39:48,802 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/fa3ec7c48aaf4201b96ebbf3e495c9e5 is 50, key is test_row_0/B:col10/1732505986358/Put/seqid=0 2024-11-25T03:39:48,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742422_1598 (size=9857) 2024-11-25T03:39:49,205 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/fa3ec7c48aaf4201b96ebbf3e495c9e5 2024-11-25T03:39:49,210 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/771e8c0a1d6a40f0aa373a0b072fe3b5 is 50, key is test_row_0/C:col10/1732505986358/Put/seqid=0 2024-11-25T03:39:49,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742423_1599 (size=9857) 2024-11-25T03:39:49,613 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/771e8c0a1d6a40f0aa373a0b072fe3b5 2024-11-25T03:39:49,616 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/A/ec8a135e774d435281748c179d9ac1a8 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/ec8a135e774d435281748c179d9ac1a8 2024-11-25T03:39:49,619 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/ec8a135e774d435281748c179d9ac1a8, entries=100, sequenceid=363, filesize=22.0 K 2024-11-25T03:39:49,620 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/B/fa3ec7c48aaf4201b96ebbf3e495c9e5 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/fa3ec7c48aaf4201b96ebbf3e495c9e5 2024-11-25T03:39:49,622 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/fa3ec7c48aaf4201b96ebbf3e495c9e5, entries=100, sequenceid=363, filesize=9.6 K 2024-11-25T03:39:49,623 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/.tmp/C/771e8c0a1d6a40f0aa373a0b072fe3b5 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/771e8c0a1d6a40f0aa373a0b072fe3b5 2024-11-25T03:39:49,625 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/771e8c0a1d6a40f0aa373a0b072fe3b5, entries=100, sequenceid=363, filesize=9.6 K 2024-11-25T03:39:49,626 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 33de131a28520cc9ecc2bf159dfd6128 in 1642ms, sequenceid=363, compaction requested=true 2024-11-25T03:39:49,626 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/6ad3d64a9c404233b1730e18f65d2024, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/7f1037a2539447409d0156d2b5b4d7a9, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/37f8a4d8e85d4da9bae5d50c5e33193d, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/7a2c3e4403304167b467fb0e0d5fa474, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/76696f1b1d944ec79bffafc7767981c5, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/0d2e09f840fc427b8dcaad02fb273330, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/52100752fcf04c22bc0cad7795bbefe8, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/1dc11343b8cb40f29142a0ffb3ffc46f, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/15e329e39f2b4f41bb7befe87e6e17b6, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/98b96364bccd453197ee21c6a15b980f, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/6bbf8ad142ed43c3b8a20e74c8517dac, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/d779162afa5e4da99c27f41d37b50207, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/792badc607b844d4901bbfe457eee763, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/cec7a41432974390a4976d3c80d3331f, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/d87d1802b56943acae57e81538fdf741, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/f9658e0b513640c98b7905a692baf416, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/0abfba6efe304615915f1d6bcce7f420, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/7523289aec1f4bd7a2b48735b8ab96db, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/02ae6f0eb345452f92bb1aa7aa0b3ad4, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/a5376abb18444e42b5f82b02a260294e, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/c16eb552f40b4bf7a9d9e9b0f323574d, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/c5e3f8aa97fe4068818cba6b72e484c1] to archive 2024-11-25T03:39:49,627 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T03:39:49,628 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/6ad3d64a9c404233b1730e18f65d2024 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/6ad3d64a9c404233b1730e18f65d2024 2024-11-25T03:39:49,628 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/7f1037a2539447409d0156d2b5b4d7a9 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/7f1037a2539447409d0156d2b5b4d7a9 2024-11-25T03:39:49,629 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/37f8a4d8e85d4da9bae5d50c5e33193d to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/37f8a4d8e85d4da9bae5d50c5e33193d 2024-11-25T03:39:49,630 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/7a2c3e4403304167b467fb0e0d5fa474 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/7a2c3e4403304167b467fb0e0d5fa474 2024-11-25T03:39:49,631 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/76696f1b1d944ec79bffafc7767981c5 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/76696f1b1d944ec79bffafc7767981c5 2024-11-25T03:39:49,631 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/0d2e09f840fc427b8dcaad02fb273330 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/0d2e09f840fc427b8dcaad02fb273330 2024-11-25T03:39:49,634 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/52100752fcf04c22bc0cad7795bbefe8 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/52100752fcf04c22bc0cad7795bbefe8 2024-11-25T03:39:49,634 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/1dc11343b8cb40f29142a0ffb3ffc46f to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/1dc11343b8cb40f29142a0ffb3ffc46f 2024-11-25T03:39:49,635 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/15e329e39f2b4f41bb7befe87e6e17b6 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/15e329e39f2b4f41bb7befe87e6e17b6 2024-11-25T03:39:49,636 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/98b96364bccd453197ee21c6a15b980f to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/98b96364bccd453197ee21c6a15b980f 2024-11-25T03:39:49,637 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/6bbf8ad142ed43c3b8a20e74c8517dac to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/6bbf8ad142ed43c3b8a20e74c8517dac 2024-11-25T03:39:49,638 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/d779162afa5e4da99c27f41d37b50207 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/d779162afa5e4da99c27f41d37b50207 2024-11-25T03:39:49,639 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/792badc607b844d4901bbfe457eee763 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/792badc607b844d4901bbfe457eee763 2024-11-25T03:39:49,640 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/cec7a41432974390a4976d3c80d3331f to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/cec7a41432974390a4976d3c80d3331f 2024-11-25T03:39:49,641 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/d87d1802b56943acae57e81538fdf741 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/d87d1802b56943acae57e81538fdf741 2024-11-25T03:39:49,642 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/f9658e0b513640c98b7905a692baf416 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/f9658e0b513640c98b7905a692baf416 2024-11-25T03:39:49,643 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/0abfba6efe304615915f1d6bcce7f420 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/0abfba6efe304615915f1d6bcce7f420 2024-11-25T03:39:49,644 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/7523289aec1f4bd7a2b48735b8ab96db to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/7523289aec1f4bd7a2b48735b8ab96db 2024-11-25T03:39:49,645 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/02ae6f0eb345452f92bb1aa7aa0b3ad4 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/02ae6f0eb345452f92bb1aa7aa0b3ad4 2024-11-25T03:39:49,646 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/a5376abb18444e42b5f82b02a260294e to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/a5376abb18444e42b5f82b02a260294e 2024-11-25T03:39:49,647 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/c16eb552f40b4bf7a9d9e9b0f323574d to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/c16eb552f40b4bf7a9d9e9b0f323574d 2024-11-25T03:39:49,648 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/c5e3f8aa97fe4068818cba6b72e484c1 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/c5e3f8aa97fe4068818cba6b72e484c1 2024-11-25T03:39:49,649 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/d154319122014d238fc3a0508af7c668, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/b4c6ba40e98f4b6f94c8974a07cc1198, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/0aac5e69dbfc456e90d83c14af3dfb28, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/370b7b0db6414865bb62c81e694ce820, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/41729ae2ea664b0f8333b936edd5991e, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/07242c03c4b64266aeb21c0202339dfe, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/2835373f0ea3427b9477659423dfbdc5, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/34e47fb9708546c4acd1b55de485897b, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/6f831653fa694f90b2e469d85fee9dc4, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/26724993e6b9468d938aac8e6bbe5f56, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/59248e7703bc44268d4d3218765d0235, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/5ff203514bb84f55b4e3172dc4fda78a, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/52a0bc1b3f374d8fb5f1be373f6a5025, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/6b07051d69f4480f9d601447feed3ca4, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/79b9057170a94f8192c650f7d2af237f, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/463e870103f142389c146c3e8c1367fa, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/96a3dd182d574258b667a6597a7b79c2, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/675d04b0aa52473a993e7d03de69c7d8, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/d2f61736d23147a28aa31993ae67c20e, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/bf648520d7924bf191fa7da414b187c7, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/e5db130b3a364fceb212e60ac16107c8, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/cd76d562e26d42b38346a4151b88a33c] to archive 2024-11-25T03:39:49,650 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T03:39:49,651 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/d154319122014d238fc3a0508af7c668 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/d154319122014d238fc3a0508af7c668 2024-11-25T03:39:49,652 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/b4c6ba40e98f4b6f94c8974a07cc1198 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/b4c6ba40e98f4b6f94c8974a07cc1198 2024-11-25T03:39:49,653 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/0aac5e69dbfc456e90d83c14af3dfb28 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/0aac5e69dbfc456e90d83c14af3dfb28 2024-11-25T03:39:49,654 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/370b7b0db6414865bb62c81e694ce820 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/370b7b0db6414865bb62c81e694ce820 2024-11-25T03:39:49,655 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/41729ae2ea664b0f8333b936edd5991e to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/41729ae2ea664b0f8333b936edd5991e 2024-11-25T03:39:49,656 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/07242c03c4b64266aeb21c0202339dfe to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/07242c03c4b64266aeb21c0202339dfe 2024-11-25T03:39:49,657 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/2835373f0ea3427b9477659423dfbdc5 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/2835373f0ea3427b9477659423dfbdc5 2024-11-25T03:39:49,658 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/34e47fb9708546c4acd1b55de485897b to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/34e47fb9708546c4acd1b55de485897b 2024-11-25T03:39:49,659 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/6f831653fa694f90b2e469d85fee9dc4 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/6f831653fa694f90b2e469d85fee9dc4 2024-11-25T03:39:49,659 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/26724993e6b9468d938aac8e6bbe5f56 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/26724993e6b9468d938aac8e6bbe5f56 2024-11-25T03:39:49,660 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/59248e7703bc44268d4d3218765d0235 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/59248e7703bc44268d4d3218765d0235 2024-11-25T03:39:49,661 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/5ff203514bb84f55b4e3172dc4fda78a to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/5ff203514bb84f55b4e3172dc4fda78a 2024-11-25T03:39:49,663 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/52a0bc1b3f374d8fb5f1be373f6a5025 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/52a0bc1b3f374d8fb5f1be373f6a5025 2024-11-25T03:39:49,664 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/6b07051d69f4480f9d601447feed3ca4 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/6b07051d69f4480f9d601447feed3ca4 2024-11-25T03:39:49,665 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/79b9057170a94f8192c650f7d2af237f to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/79b9057170a94f8192c650f7d2af237f 2024-11-25T03:39:49,666 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/463e870103f142389c146c3e8c1367fa to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/463e870103f142389c146c3e8c1367fa 2024-11-25T03:39:49,667 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/96a3dd182d574258b667a6597a7b79c2 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/96a3dd182d574258b667a6597a7b79c2 2024-11-25T03:39:49,668 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/675d04b0aa52473a993e7d03de69c7d8 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/675d04b0aa52473a993e7d03de69c7d8 2024-11-25T03:39:49,669 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/d2f61736d23147a28aa31993ae67c20e to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/d2f61736d23147a28aa31993ae67c20e 2024-11-25T03:39:49,670 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/bf648520d7924bf191fa7da414b187c7 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/bf648520d7924bf191fa7da414b187c7 2024-11-25T03:39:49,671 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/e5db130b3a364fceb212e60ac16107c8 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/e5db130b3a364fceb212e60ac16107c8 2024-11-25T03:39:49,671 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/cd76d562e26d42b38346a4151b88a33c to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/cd76d562e26d42b38346a4151b88a33c 2024-11-25T03:39:49,673 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/fe66b944adce47ee9dd2530b2fec1106, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/c6d87f70a45e42a48877552edc5687d3, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/b37f2e9ed0bd4a7b8f0352a0bf29ad00, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/7ec138df5c644e4f80ad22f23e0299f7, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/c861c53dbd2541feb7352d641b84d007, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/6976775bf9884734a52a28d750e904c2, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/38deacbe2872454ba52d56e687e8e3f3, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/9017ce44b4364640b1eddc38b54c8738, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/593f2d6d4452449ab090f748a93b917a, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/4fcae415bc81404d823b80aa7f0cd78a, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/cd10c323b7364cc68f9e3c8333ef2d1e, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/9d0861a43b1c47a8ba7018316825fe3c, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/3698fd24b35d461ab1fdc9431f1fe424, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/7797950a3c944ba2884987fff15fb5f3, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/19afaf52d82d45eea862cd5dbaf727a0, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/993d81635c3d4b2e88099e8ed6745d5c, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/33e18398bcbe40f8a03a7e2e3b907a7d, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/b1ee2ed0769f4eab80bb286bcf25d77f, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/d00fcd9ff0fb40a9ae8c284df19a6e5b, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/3bf041c4ac1f493f9db2e7e2109b93e7, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/855577da0db044528d6beb50bcc2b622, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/54dfa070984a45dda35e89303264c465] to archive 2024-11-25T03:39:49,674 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T03:39:49,675 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/fe66b944adce47ee9dd2530b2fec1106 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/fe66b944adce47ee9dd2530b2fec1106 2024-11-25T03:39:49,676 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/c6d87f70a45e42a48877552edc5687d3 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/c6d87f70a45e42a48877552edc5687d3 2024-11-25T03:39:49,677 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/b37f2e9ed0bd4a7b8f0352a0bf29ad00 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/b37f2e9ed0bd4a7b8f0352a0bf29ad00 2024-11-25T03:39:49,678 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/7ec138df5c644e4f80ad22f23e0299f7 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/7ec138df5c644e4f80ad22f23e0299f7 2024-11-25T03:39:49,679 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/c861c53dbd2541feb7352d641b84d007 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/c861c53dbd2541feb7352d641b84d007 2024-11-25T03:39:49,680 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/6976775bf9884734a52a28d750e904c2 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/6976775bf9884734a52a28d750e904c2 2024-11-25T03:39:49,681 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/38deacbe2872454ba52d56e687e8e3f3 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/38deacbe2872454ba52d56e687e8e3f3 2024-11-25T03:39:49,682 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/9017ce44b4364640b1eddc38b54c8738 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/9017ce44b4364640b1eddc38b54c8738 2024-11-25T03:39:49,683 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/593f2d6d4452449ab090f748a93b917a to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/593f2d6d4452449ab090f748a93b917a 2024-11-25T03:39:49,684 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/4fcae415bc81404d823b80aa7f0cd78a to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/4fcae415bc81404d823b80aa7f0cd78a 2024-11-25T03:39:49,685 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/cd10c323b7364cc68f9e3c8333ef2d1e to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/cd10c323b7364cc68f9e3c8333ef2d1e 2024-11-25T03:39:49,685 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/9d0861a43b1c47a8ba7018316825fe3c to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/9d0861a43b1c47a8ba7018316825fe3c 2024-11-25T03:39:49,686 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/3698fd24b35d461ab1fdc9431f1fe424 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/3698fd24b35d461ab1fdc9431f1fe424 2024-11-25T03:39:49,687 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/7797950a3c944ba2884987fff15fb5f3 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/7797950a3c944ba2884987fff15fb5f3 2024-11-25T03:39:49,687 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/19afaf52d82d45eea862cd5dbaf727a0 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/19afaf52d82d45eea862cd5dbaf727a0 2024-11-25T03:39:49,688 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/993d81635c3d4b2e88099e8ed6745d5c to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/993d81635c3d4b2e88099e8ed6745d5c 2024-11-25T03:39:49,689 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/33e18398bcbe40f8a03a7e2e3b907a7d to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/33e18398bcbe40f8a03a7e2e3b907a7d 2024-11-25T03:39:49,690 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/b1ee2ed0769f4eab80bb286bcf25d77f to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/b1ee2ed0769f4eab80bb286bcf25d77f 2024-11-25T03:39:49,691 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/d00fcd9ff0fb40a9ae8c284df19a6e5b to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/d00fcd9ff0fb40a9ae8c284df19a6e5b 2024-11-25T03:39:49,692 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/3bf041c4ac1f493f9db2e7e2109b93e7 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/3bf041c4ac1f493f9db2e7e2109b93e7 2024-11-25T03:39:49,693 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/855577da0db044528d6beb50bcc2b622 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/855577da0db044528d6beb50bcc2b622 2024-11-25T03:39:49,693 DEBUG [StoreCloser-TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/54dfa070984a45dda35e89303264c465 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/54dfa070984a45dda35e89303264c465 2024-11-25T03:39:49,697 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/recovered.edits/366.seqid, newMaxSeqId=366, maxSeqId=4 2024-11-25T03:39:49,698 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. 2024-11-25T03:39:49,698 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1676): Region close journal for 33de131a28520cc9ecc2bf159dfd6128: Waiting for close lock at 1732505986519Running coprocessor pre-close hooks at 1732505986519Disabling compacts and flushes for region at 1732505986519Disabling writes for close at 1732505987983 (+1464 ms)Obtaining lock to block concurrent updates at 1732505987983Preparing flush snapshotting stores in 33de131a28520cc9ecc2bf159dfd6128 at 1732505987983Finished memstore snapshotting TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1732505987983Flushing stores of TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128. at 1732505987984 (+1 ms)Flushing 33de131a28520cc9ecc2bf159dfd6128/A: creating writer at 1732505987984Flushing mob file 33de131a28520cc9ecc2bf159dfd6128/A: appending metadata at 1732505987987 (+3 ms)Flushing mob file 33de131a28520cc9ecc2bf159dfd6128/A: closing flushed file at 1732505987987Flushing 33de131a28520cc9ecc2bf159dfd6128/A: appending metadata at 1732505988393 (+406 ms)Flushing 33de131a28520cc9ecc2bf159dfd6128/A: closing flushed file at 1732505988394 (+1 ms)Flushing 33de131a28520cc9ecc2bf159dfd6128/B: creating writer at 1732505988800 (+406 ms)Flushing 33de131a28520cc9ecc2bf159dfd6128/B: appending metadata at 1732505988802 (+2 ms)Flushing 33de131a28520cc9ecc2bf159dfd6128/B: closing flushed file at 1732505988802Flushing 33de131a28520cc9ecc2bf159dfd6128/C: creating writer at 1732505989208 (+406 ms)Flushing 33de131a28520cc9ecc2bf159dfd6128/C: appending metadata at 1732505989210 (+2 ms)Flushing 33de131a28520cc9ecc2bf159dfd6128/C: closing flushed file at 1732505989210Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@634009d3: reopening flushed file at 1732505989616 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c2066e9: reopening flushed file at 1732505989619 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@27f9a9ae: reopening flushed file at 1732505989622 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 33de131a28520cc9ecc2bf159dfd6128 in 1642ms, sequenceid=363, compaction requested=true at 1732505989626 (+4 ms)Writing region close event to WAL at 1732505989694 (+68 ms)Running coprocessor post-close hooks at 1732505989698 (+4 ms)Closed at 1732505989698 2024-11-25T03:39:49,699 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(157): Closed 33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:49,700 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=33de131a28520cc9ecc2bf159dfd6128, regionState=CLOSED 2024-11-25T03:39:49,701 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure 33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 because future has completed 2024-11-25T03:39:49,703 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-25T03:39:49,703 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseRegionProcedure 33de131a28520cc9ecc2bf159dfd6128, server=38a5d2077268,34881,1732505806588 in 3.3350 sec 2024-11-25T03:39:49,704 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=164, resume processing ppid=163 2024-11-25T03:39:49,704 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, ppid=163, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=33de131a28520cc9ecc2bf159dfd6128, UNASSIGN in 3.3380 sec 2024-11-25T03:39:49,706 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-25T03:39:49,706 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 3.3410 sec 2024-11-25T03:39:49,708 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732505989708"}]},"ts":"1732505989708"} 2024-11-25T03:39:49,709 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-25T03:39:49,709 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-25T03:39:49,711 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 3.3500 sec 2024-11-25T03:39:50,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-25T03:39:50,518 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:39:50,518 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-25T03:39:50,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:39:50,520 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:39:50,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-25T03:39:50,520 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=166, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:39:50,522 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:50,523 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A, FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B, FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C, FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/recovered.edits] 2024-11-25T03:39:50,525 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/518dbe69c7bc4770ab9c7601a3783373 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/518dbe69c7bc4770ab9c7601a3783373 2024-11-25T03:39:50,526 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/ec8a135e774d435281748c179d9ac1a8 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/ec8a135e774d435281748c179d9ac1a8 2024-11-25T03:39:50,527 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/fafab48df4e34d68bca8e986b308425a to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/A/fafab48df4e34d68bca8e986b308425a 2024-11-25T03:39:50,529 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/6cbfee9aef6e47ab95eb0219239d0ac6 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/6cbfee9aef6e47ab95eb0219239d0ac6 2024-11-25T03:39:50,529 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/dc962066135347938ee9f535575f15cb to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/dc962066135347938ee9f535575f15cb 2024-11-25T03:39:50,530 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/fa3ec7c48aaf4201b96ebbf3e495c9e5 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/B/fa3ec7c48aaf4201b96ebbf3e495c9e5 2024-11-25T03:39:50,532 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/150ffee76fae4735826067ee524e44d2 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/150ffee76fae4735826067ee524e44d2 2024-11-25T03:39:50,533 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/771e8c0a1d6a40f0aa373a0b072fe3b5 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/771e8c0a1d6a40f0aa373a0b072fe3b5 2024-11-25T03:39:50,534 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/7755be0a89b648748eece004c886193d to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/C/7755be0a89b648748eece004c886193d 2024-11-25T03:39:50,537 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/recovered.edits/366.seqid to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128/recovered.edits/366.seqid 2024-11-25T03:39:50,537 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/default/TestAcidGuarantees/33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:50,537 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-25T03:39:50,538 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-25T03:39:50,538 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-25T03:39:50,541 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125003e24526fb5428688a7b543fa329882_33de131a28520cc9ecc2bf159dfd6128 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125003e24526fb5428688a7b543fa329882_33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:50,542 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125012476379ece44a8a88571c5e5c066f2_33de131a28520cc9ecc2bf159dfd6128 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125012476379ece44a8a88571c5e5c066f2_33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:50,543 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112508f28a9268044bac92570f0c47fedbef_33de131a28520cc9ecc2bf159dfd6128 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112508f28a9268044bac92570f0c47fedbef_33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:50,545 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112521d11eeeff8c4d4aa5a765cebad00866_33de131a28520cc9ecc2bf159dfd6128 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112521d11eeeff8c4d4aa5a765cebad00866_33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:50,546 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411253adf353dd3bd47a490b701b98351541c_33de131a28520cc9ecc2bf159dfd6128 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411253adf353dd3bd47a490b701b98351541c_33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:50,547 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411255aa2b0259f0341f794e63d086b70ae87_33de131a28520cc9ecc2bf159dfd6128 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411255aa2b0259f0341f794e63d086b70ae87_33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:50,547 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411255aa8c766a3be448aa56fb93d7ca88509_33de131a28520cc9ecc2bf159dfd6128 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411255aa8c766a3be448aa56fb93d7ca88509_33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:50,548 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125615ccdfc13ad42ea83536bb8f6a4d5e8_33de131a28520cc9ecc2bf159dfd6128 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125615ccdfc13ad42ea83536bb8f6a4d5e8_33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:50,549 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411256d4279ebae174215bcb178cea4390ce3_33de131a28520cc9ecc2bf159dfd6128 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411256d4279ebae174215bcb178cea4390ce3_33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:50,550 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411256e057fb211f644d190cbacad6144ae7c_33de131a28520cc9ecc2bf159dfd6128 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411256e057fb211f644d190cbacad6144ae7c_33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:50,551 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112570f37f2a9ed04efba5dd580e64293bfb_33de131a28520cc9ecc2bf159dfd6128 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112570f37f2a9ed04efba5dd580e64293bfb_33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:50,552 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112585c554dce98544c380913606389c018b_33de131a28520cc9ecc2bf159dfd6128 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112585c554dce98544c380913606389c018b_33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:50,552 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112585db22c7b31840c486c0f140823bd74d_33de131a28520cc9ecc2bf159dfd6128 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112585db22c7b31840c486c0f140823bd74d_33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:50,553 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125a13bcf0bd9d2414295b9cea669c54580_33de131a28520cc9ecc2bf159dfd6128 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125a13bcf0bd9d2414295b9cea669c54580_33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:50,554 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125acfb2def46de45de9d57e83fe0c2e1e8_33de131a28520cc9ecc2bf159dfd6128 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125acfb2def46de45de9d57e83fe0c2e1e8_33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:50,555 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125b220086c25824c8d8ab02146331abcc5_33de131a28520cc9ecc2bf159dfd6128 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125b220086c25824c8d8ab02146331abcc5_33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:50,555 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125d16cf89c59f94733a7ce9be54a8905ce_33de131a28520cc9ecc2bf159dfd6128 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125d16cf89c59f94733a7ce9be54a8905ce_33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:50,556 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125d51f1e0e1066448b8babcc37278a4592_33de131a28520cc9ecc2bf159dfd6128 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125d51f1e0e1066448b8babcc37278a4592_33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:50,557 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125dc72c12b206644a6957a02dd15314092_33de131a28520cc9ecc2bf159dfd6128 to hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125dc72c12b206644a6957a02dd15314092_33de131a28520cc9ecc2bf159dfd6128 2024-11-25T03:39:50,557 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-25T03:39:50,559 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=166, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:39:50,561 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-25T03:39:50,563 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-25T03:39:50,563 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=166, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:39:50,563 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-25T03:39:50,564 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732505990563"}]},"ts":"9223372036854775807"} 2024-11-25T03:39:50,565 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-25T03:39:50,565 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 33de131a28520cc9ecc2bf159dfd6128, NAME => 'TestAcidGuarantees,,1732505961755.33de131a28520cc9ecc2bf159dfd6128.', STARTKEY => '', ENDKEY => ''}] 2024-11-25T03:39:50,565 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-25T03:39:50,565 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732505990565"}]},"ts":"9223372036854775807"} 2024-11-25T03:39:50,566 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-25T03:39:50,567 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=166, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T03:39:50,568 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 49 msec 2024-11-25T03:39:50,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-25T03:39:50,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-25T03:39:50,628 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-25T03:39:50,637 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=241 (was 238) - Thread LEAK? -, OpenFileDescriptor=455 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=709 (was 911), ProcessCount=11 (was 11), AvailableMemoryMB=6515 (was 6639) 2024-11-25T03:39:50,637 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-25T03:39:50,637 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T03:39:50,637 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T03:39:50,637 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:50,638 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:50,638 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:39:50,638 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-25T03:39:50,638 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1107072306, stopped=false 2024-11-25T03:39:50,638 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=38a5d2077268,42243,1732505805780 2024-11-25T03:39:50,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42243-0x1014d328acf0000, quorum=127.0.0.1:63582, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T03:39:50,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34881-0x1014d328acf0001, quorum=127.0.0.1:63582, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T03:39:50,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42243-0x1014d328acf0000, quorum=127.0.0.1:63582, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:39:50,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34881-0x1014d328acf0001, quorum=127.0.0.1:63582, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:39:50,640 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:39:50,641 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T03:39:50,641 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T03:39:50,641 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:50,641 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34881-0x1014d328acf0001, quorum=127.0.0.1:63582, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T03:39:50,641 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42243-0x1014d328acf0000, quorum=127.0.0.1:63582, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T03:39:50,641 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '38a5d2077268,34881,1732505806588' ***** 2024-11-25T03:39:50,641 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-25T03:39:50,642 INFO [RS:0;38a5d2077268:34881 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-25T03:39:50,642 INFO [RS:0;38a5d2077268:34881 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-25T03:39:50,642 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-25T03:39:50,642 INFO [RS:0;38a5d2077268:34881 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-25T03:39:50,642 INFO [RS:0;38a5d2077268:34881 {}] regionserver.HRegionServer(959): stopping server 38a5d2077268,34881,1732505806588 2024-11-25T03:39:50,642 INFO [RS:0;38a5d2077268:34881 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T03:39:50,642 INFO [RS:0;38a5d2077268:34881 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;38a5d2077268:34881. 2024-11-25T03:39:50,642 DEBUG [RS:0;38a5d2077268:34881 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T03:39:50,642 DEBUG [RS:0;38a5d2077268:34881 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:50,642 INFO [RS:0;38a5d2077268:34881 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-25T03:39:50,642 INFO [RS:0;38a5d2077268:34881 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-25T03:39:50,642 INFO [RS:0;38a5d2077268:34881 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-25T03:39:50,643 INFO [RS:0;38a5d2077268:34881 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-25T03:39:50,643 INFO [RS:0;38a5d2077268:34881 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-25T03:39:50,643 DEBUG [RS:0;38a5d2077268:34881 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-25T03:39:50,644 DEBUG [RS:0;38a5d2077268:34881 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T03:39:50,644 DEBUG [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-25T03:39:50,644 INFO [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-25T03:39:50,644 DEBUG [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-25T03:39:50,644 DEBUG [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-25T03:39:50,644 DEBUG [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-25T03:39:50,644 INFO [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-25T03:39:50,667 DEBUG [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/hbase/meta/1588230740/.tmp/info/333535639ac5418fbb604dda51ab1a56 is 95, key is TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857./info:/1732505837164/DeleteFamily/seqid=0 2024-11-25T03:39:50,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742424_1600 (size=6514) 2024-11-25T03:39:50,724 INFO [regionserver/38a5d2077268:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T03:39:50,754 INFO [regionserver/38a5d2077268:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-25T03:39:50,754 INFO [regionserver/38a5d2077268:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-25T03:39:50,844 DEBUG [RS:0;38a5d2077268:34881 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T03:39:51,044 DEBUG [RS:0;38a5d2077268:34881 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T03:39:51,070 INFO [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/hbase/meta/1588230740/.tmp/info/333535639ac5418fbb604dda51ab1a56 2024-11-25T03:39:51,088 DEBUG [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/hbase/meta/1588230740/.tmp/ns/a9025f3b37434bf6942320a739255625 is 93, key is TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857./ns:/1732505837124/DeleteFamily/seqid=0 2024-11-25T03:39:51,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742425_1601 (size=5906) 2024-11-25T03:39:51,244 DEBUG [RS:0;38a5d2077268:34881 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T03:39:51,444 DEBUG [RS:0;38a5d2077268:34881 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T03:39:51,491 INFO [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/hbase/meta/1588230740/.tmp/ns/a9025f3b37434bf6942320a739255625 2024-11-25T03:39:51,509 DEBUG [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/hbase/meta/1588230740/.tmp/rep_barrier/4701fb61b9814b11a7100eb1807be461 is 102, key is TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857./rep_barrier:/1732505837124/DeleteFamily/seqid=0 2024-11-25T03:39:51,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742426_1602 (size=6025) 2024-11-25T03:39:51,645 DEBUG [RS:0;38a5d2077268:34881 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T03:39:51,845 DEBUG [RS:0;38a5d2077268:34881 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T03:39:51,913 INFO [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/hbase/meta/1588230740/.tmp/rep_barrier/4701fb61b9814b11a7100eb1807be461 2024-11-25T03:39:51,930 DEBUG [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/hbase/meta/1588230740/.tmp/table/f3b3a4900f394e0f8f9e4f62fc96eb35 is 96, key is TestAcidGuarantees,,1732505810334.ee186277170c2bdfb070a7c209286857./table:/1732505837124/DeleteFamily/seqid=0 2024-11-25T03:39:51,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742427_1603 (size=5936) 2024-11-25T03:39:52,045 DEBUG [RS:0;38a5d2077268:34881 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T03:39:52,245 DEBUG [RS:0;38a5d2077268:34881 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T03:39:52,334 INFO [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/hbase/meta/1588230740/.tmp/table/f3b3a4900f394e0f8f9e4f62fc96eb35 2024-11-25T03:39:52,337 DEBUG [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/hbase/meta/1588230740/.tmp/info/333535639ac5418fbb604dda51ab1a56 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/hbase/meta/1588230740/info/333535639ac5418fbb604dda51ab1a56 2024-11-25T03:39:52,339 INFO [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/hbase/meta/1588230740/info/333535639ac5418fbb604dda51ab1a56, entries=12, sequenceid=90, filesize=6.4 K 2024-11-25T03:39:52,340 DEBUG [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/hbase/meta/1588230740/.tmp/ns/a9025f3b37434bf6942320a739255625 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/hbase/meta/1588230740/ns/a9025f3b37434bf6942320a739255625 2024-11-25T03:39:52,342 INFO [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/hbase/meta/1588230740/ns/a9025f3b37434bf6942320a739255625, entries=8, sequenceid=90, filesize=5.8 K 2024-11-25T03:39:52,343 DEBUG [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/hbase/meta/1588230740/.tmp/rep_barrier/4701fb61b9814b11a7100eb1807be461 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/hbase/meta/1588230740/rep_barrier/4701fb61b9814b11a7100eb1807be461 2024-11-25T03:39:52,345 INFO [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/hbase/meta/1588230740/rep_barrier/4701fb61b9814b11a7100eb1807be461, entries=6, sequenceid=90, filesize=5.9 K 2024-11-25T03:39:52,345 DEBUG [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/hbase/meta/1588230740/.tmp/table/f3b3a4900f394e0f8f9e4f62fc96eb35 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/hbase/meta/1588230740/table/f3b3a4900f394e0f8f9e4f62fc96eb35 2024-11-25T03:39:52,348 INFO [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/hbase/meta/1588230740/table/f3b3a4900f394e0f8f9e4f62fc96eb35, entries=7, sequenceid=90, filesize=5.8 K 2024-11-25T03:39:52,348 INFO [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1704ms, sequenceid=90, compaction requested=false 2024-11-25T03:39:52,352 DEBUG [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-25T03:39:52,352 DEBUG [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-25T03:39:52,352 INFO [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-25T03:39:52,352 DEBUG [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732505990643Running coprocessor pre-close hooks at 1732505990644 (+1 ms)Disabling compacts and flushes for region at 1732505990644Disabling writes for close at 1732505990644Obtaining lock to block concurrent updates at 1732505990644Preparing flush snapshotting stores in 1588230740 at 1732505990644Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732505990644Flushing stores of hbase:meta,,1.1588230740 at 1732505990644Flushing 1588230740/info: creating writer at 1732505990645 (+1 ms)Flushing 1588230740/info: appending metadata at 1732505990666 (+21 ms)Flushing 1588230740/info: closing flushed file at 1732505990666Flushing 1588230740/ns: creating writer at 1732505991073 (+407 ms)Flushing 1588230740/ns: appending metadata at 1732505991088 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1732505991088Flushing 1588230740/rep_barrier: creating writer at 1732505991494 (+406 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732505991509 (+15 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732505991509Flushing 1588230740/table: creating writer at 1732505991916 (+407 ms)Flushing 1588230740/table: appending metadata at 1732505991930 (+14 ms)Flushing 1588230740/table: closing flushed file at 1732505991930Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39e5d376: reopening flushed file at 1732505992337 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8325127: reopening flushed file at 1732505992340 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6374d6ac: reopening flushed file at 1732505992342 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53f7cb7c: reopening flushed file at 1732505992345 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1704ms, sequenceid=90, compaction requested=false at 1732505992348 (+3 ms)Writing region close event to WAL at 1732505992350 (+2 ms)Running coprocessor post-close hooks at 1732505992352 (+2 ms)Closed at 1732505992352 2024-11-25T03:39:52,353 DEBUG [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-25T03:39:52,445 INFO [RS:0;38a5d2077268:34881 {}] regionserver.HRegionServer(976): stopping server 38a5d2077268,34881,1732505806588; all regions closed. 2024-11-25T03:39:52,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741834_1010 (size=25344) 2024-11-25T03:39:52,451 DEBUG [RS:0;38a5d2077268:34881 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/oldWALs 2024-11-25T03:39:52,451 INFO [RS:0;38a5d2077268:34881 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 38a5d2077268%2C34881%2C1732505806588.meta:.meta(num 1732505809829) 2024-11-25T03:39:52,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741832_1008 (size=13809274) 2024-11-25T03:39:52,454 DEBUG [RS:0;38a5d2077268:34881 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/oldWALs 2024-11-25T03:39:52,454 INFO [RS:0;38a5d2077268:34881 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 38a5d2077268%2C34881%2C1732505806588:(num 1732505808918) 2024-11-25T03:39:52,454 DEBUG [RS:0;38a5d2077268:34881 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:52,454 INFO [RS:0;38a5d2077268:34881 {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T03:39:52,454 INFO [RS:0;38a5d2077268:34881 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T03:39:52,454 INFO [RS:0;38a5d2077268:34881 {}] hbase.ChoreService(370): Chore service for: regionserver/38a5d2077268:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-25T03:39:52,455 INFO [RS:0;38a5d2077268:34881 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T03:39:52,455 INFO [regionserver/38a5d2077268:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T03:39:52,455 INFO [RS:0;38a5d2077268:34881 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34881 2024-11-25T03:39:52,459 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34881-0x1014d328acf0001, quorum=127.0.0.1:63582, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/38a5d2077268,34881,1732505806588 2024-11-25T03:39:52,459 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42243-0x1014d328acf0000, quorum=127.0.0.1:63582, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T03:39:52,459 INFO [RS:0;38a5d2077268:34881 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T03:39:52,461 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [38a5d2077268,34881,1732505806588] 2024-11-25T03:39:52,467 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/38a5d2077268,34881,1732505806588 already deleted, retry=false 2024-11-25T03:39:52,468 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 38a5d2077268,34881,1732505806588 expired; onlineServers=0 2024-11-25T03:39:52,468 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '38a5d2077268,42243,1732505805780' ***** 2024-11-25T03:39:52,468 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-25T03:39:52,468 INFO [M:0;38a5d2077268:42243 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T03:39:52,468 INFO [M:0;38a5d2077268:42243 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T03:39:52,468 DEBUG [M:0;38a5d2077268:42243 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-25T03:39:52,468 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-25T03:39:52,468 DEBUG [M:0;38a5d2077268:42243 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-25T03:39:52,468 DEBUG [master/38a5d2077268:0:becomeActiveMaster-HFileCleaner.large.0-1732505808553 {}] cleaner.HFileCleaner(306): Exit Thread[master/38a5d2077268:0:becomeActiveMaster-HFileCleaner.large.0-1732505808553,5,FailOnTimeoutGroup] 2024-11-25T03:39:52,468 DEBUG [master/38a5d2077268:0:becomeActiveMaster-HFileCleaner.small.0-1732505808557 {}] cleaner.HFileCleaner(306): Exit Thread[master/38a5d2077268:0:becomeActiveMaster-HFileCleaner.small.0-1732505808557,5,FailOnTimeoutGroup] 2024-11-25T03:39:52,468 INFO [M:0;38a5d2077268:42243 {}] hbase.ChoreService(370): Chore service for: master/38a5d2077268:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-25T03:39:52,468 INFO [M:0;38a5d2077268:42243 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T03:39:52,469 DEBUG [M:0;38a5d2077268:42243 {}] master.HMaster(1795): Stopping service threads 2024-11-25T03:39:52,469 INFO [M:0;38a5d2077268:42243 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-25T03:39:52,469 INFO [M:0;38a5d2077268:42243 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:39:52,469 ERROR [M:0;38a5d2077268:42243 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-25T03:39:52,469 INFO [M:0;38a5d2077268:42243 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-25T03:39:52,470 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-25T03:39:52,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42243-0x1014d328acf0000, quorum=127.0.0.1:63582, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-25T03:39:52,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42243-0x1014d328acf0000, quorum=127.0.0.1:63582, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:39:52,470 DEBUG [M:0;38a5d2077268:42243 {}] zookeeper.ZKUtil(347): master:42243-0x1014d328acf0000, quorum=127.0.0.1:63582, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-25T03:39:52,470 WARN [M:0;38a5d2077268:42243 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-25T03:39:52,471 INFO [M:0;38a5d2077268:42243 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/.lastflushedseqids 2024-11-25T03:39:52,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742428_1604 (size=72) 2024-11-25T03:39:52,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34881-0x1014d328acf0001, quorum=127.0.0.1:63582, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T03:39:52,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34881-0x1014d328acf0001, quorum=127.0.0.1:63582, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T03:39:52,562 INFO [RS:0;38a5d2077268:34881 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T03:39:52,562 INFO [RS:0;38a5d2077268:34881 {}] regionserver.HRegionServer(1031): Exiting; stopping=38a5d2077268,34881,1732505806588; zookeeper connection closed. 2024-11-25T03:39:52,562 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@47109eac {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@47109eac 2024-11-25T03:39:52,563 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-25T03:39:52,880 INFO [M:0;38a5d2077268:42243 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-25T03:39:52,880 INFO [M:0;38a5d2077268:42243 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-25T03:39:52,880 DEBUG [M:0;38a5d2077268:42243 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-25T03:39:52,881 INFO [M:0;38a5d2077268:42243 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:39:52,881 DEBUG [M:0;38a5d2077268:42243 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:39:52,881 DEBUG [M:0;38a5d2077268:42243 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-25T03:39:52,881 DEBUG [M:0;38a5d2077268:42243 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:39:52,881 INFO [M:0;38a5d2077268:42243 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=727.82 KB heapSize=892.58 KB 2024-11-25T03:39:52,896 DEBUG [M:0;38a5d2077268:42243 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/985a0c8b2f104cc7863323b87381866f is 82, key is hbase:meta,,1/info:regioninfo/1732505809928/Put/seqid=0 2024-11-25T03:39:52,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742429_1605 (size=5672) 2024-11-25T03:39:53,299 INFO [M:0;38a5d2077268:42243 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2027 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/985a0c8b2f104cc7863323b87381866f 2024-11-25T03:39:53,319 DEBUG [M:0;38a5d2077268:42243 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1b092b1f1a774852bb01a5ebfa9d7489 is 2276, key is \x00\x00\x00\x00\x00\x00\x00'/proc:d/1732505840346/Put/seqid=0 2024-11-25T03:39:53,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742430_1606 (size=43492) 2024-11-25T03:39:53,723 INFO [M:0;38a5d2077268:42243 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=727.27 KB at sequenceid=2027 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1b092b1f1a774852bb01a5ebfa9d7489 2024-11-25T03:39:53,726 INFO [M:0;38a5d2077268:42243 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1b092b1f1a774852bb01a5ebfa9d7489 2024-11-25T03:39:53,740 DEBUG [M:0;38a5d2077268:42243 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8c5aeb4297f447e3a9890d8b6bc4b4cb is 69, key is 38a5d2077268,34881,1732505806588/rs:state/1732505808661/Put/seqid=0 2024-11-25T03:39:53,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073742431_1607 (size=5156) 2024-11-25T03:39:54,144 INFO [M:0;38a5d2077268:42243 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2027 (bloomFilter=true), to=hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8c5aeb4297f447e3a9890d8b6bc4b4cb 2024-11-25T03:39:54,147 DEBUG [M:0;38a5d2077268:42243 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/985a0c8b2f104cc7863323b87381866f as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/985a0c8b2f104cc7863323b87381866f 2024-11-25T03:39:54,150 INFO [M:0;38a5d2077268:42243 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/985a0c8b2f104cc7863323b87381866f, entries=8, sequenceid=2027, filesize=5.5 K 2024-11-25T03:39:54,150 DEBUG [M:0;38a5d2077268:42243 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1b092b1f1a774852bb01a5ebfa9d7489 as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1b092b1f1a774852bb01a5ebfa9d7489 2024-11-25T03:39:54,153 INFO [M:0;38a5d2077268:42243 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1b092b1f1a774852bb01a5ebfa9d7489 2024-11-25T03:39:54,153 INFO [M:0;38a5d2077268:42243 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1b092b1f1a774852bb01a5ebfa9d7489, entries=166, sequenceid=2027, filesize=42.5 K 2024-11-25T03:39:54,153 DEBUG [M:0;38a5d2077268:42243 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8c5aeb4297f447e3a9890d8b6bc4b4cb as hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8c5aeb4297f447e3a9890d8b6bc4b4cb 2024-11-25T03:39:54,155 INFO [M:0;38a5d2077268:42243 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44505/user/jenkins/test-data/88ccf314-d496-259f-8543-ed307b4831b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8c5aeb4297f447e3a9890d8b6bc4b4cb, entries=1, sequenceid=2027, filesize=5.0 K 2024-11-25T03:39:54,156 INFO [M:0;38a5d2077268:42243 {}] regionserver.HRegion(3140): Finished flush of dataSize ~727.82 KB/745290, heapSize ~892.28 KB/913696, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1275ms, sequenceid=2027, compaction requested=false 2024-11-25T03:39:54,157 INFO [M:0;38a5d2077268:42243 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:39:54,157 DEBUG [M:0;38a5d2077268:42243 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732505992880Disabling compacts and flushes for region at 1732505992880Disabling writes for close at 1732505992881 (+1 ms)Obtaining lock to block concurrent updates at 1732505992881Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732505992881Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=745290, getHeapSize=913936, getOffHeapSize=0, getCellsCount=2218 at 1732505992881Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732505992881Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732505992881Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732505992895 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732505992896 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732505993302 (+406 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732505993318 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732505993318Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732505993726 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732505993740 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732505993740Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@668499f5: reopening flushed file at 1732505994147 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@145d763c: reopening flushed file at 1732505994150 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67cdb0b4: reopening flushed file at 1732505994153 (+3 ms)Finished flush of dataSize ~727.82 KB/745290, heapSize ~892.28 KB/913696, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1275ms, sequenceid=2027, compaction requested=false at 1732505994156 (+3 ms)Writing region close event to WAL at 1732505994157 (+1 ms)Closed at 1732505994157 2024-11-25T03:39:54,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37733 is added to blk_1073741830_1006 (size=878228) 2024-11-25T03:39:54,160 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T03:39:54,160 INFO [M:0;38a5d2077268:42243 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-25T03:39:54,160 INFO [M:0;38a5d2077268:42243 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42243 2024-11-25T03:39:54,160 INFO [M:0;38a5d2077268:42243 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T03:39:54,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42243-0x1014d328acf0000, quorum=127.0.0.1:63582, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T03:39:54,262 INFO [M:0;38a5d2077268:42243 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T03:39:54,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42243-0x1014d328acf0000, quorum=127.0.0.1:63582, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T03:39:54,267 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-25T03:39:54,269 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-25T03:39:54,269 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-25T03:39:54,269 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-25T03:39:54,269 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efa42ab5-ac11-230a-67c9-32b4726a6320/hadoop.log.dir/,STOPPED} 2024-11-25T03:39:54,272 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-25T03:39:54,272 WARN [BP-1682373972-172.17.0.2-1732505802553 heartbeating to localhost/127.0.0.1:44505 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-25T03:39:54,272 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-25T03:39:54,272 WARN [BP-1682373972-172.17.0.2-1732505802553 heartbeating to localhost/127.0.0.1:44505 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1682373972-172.17.0.2-1732505802553 (Datanode Uuid fd72fa4e-5abe-4ea1-a94d-ea69555b5c4c) service to localhost/127.0.0.1:44505 2024-11-25T03:39:54,275 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efa42ab5-ac11-230a-67c9-32b4726a6320/cluster_2bccc8e6-c56b-5b8b-fc38-b8aab8adb7e9/data/data1/current/BP-1682373972-172.17.0.2-1732505802553 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-25T03:39:54,275 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efa42ab5-ac11-230a-67c9-32b4726a6320/cluster_2bccc8e6-c56b-5b8b-fc38-b8aab8adb7e9/data/data2/current/BP-1682373972-172.17.0.2-1732505802553 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-25T03:39:54,275 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-25T03:39:54,286 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-25T03:39:54,286 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-25T03:39:54,286 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-25T03:39:54,287 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-25T03:39:54,287 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/efa42ab5-ac11-230a-67c9-32b4726a6320/hadoop.log.dir/,STOPPED} 2024-11-25T03:39:54,302 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-25T03:39:54,422 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down